projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Don't pre-allocate domain ids for si_domain
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
99126f7c
MM		 376struct dmar_domain {
377 int id; /* domain id */
4c923d47 378 int nid; /* node id */
78d8e704 379 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 380 /* bitmap of iommus this domain uses*/
99126f7c 381
c0e8a6c8
JR		 382 u16 iommu_did[DMAR_UNITS_SUPPORTED];
383 /* Domain ids per IOMMU. Use u16 since
384 * domain ids are 16 bit wide according
385 * to VT-d spec, section 9.3 */
386
00a77deb 387 struct list_head devices; /* all devices' list */
99126f7c
MM		 388 struct iova_domain iovad; /* iova's that belong to this domain */
389
390 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 391 int gaw; /* max guest address width */
392
393 /* adjusted guest address width, 0 is level 2 30-bit */
394 int agaw;
395
3b5410e7 396 int flags; /* flags to find out type of domain */
8e604097
WH		 397
398 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 399 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 400 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 401 int iommu_superpage;/* Level of superpages supported:
402 0 == 4KiB (no superpages), 1 == 2MiB,
403 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 404 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 405 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 406
407 struct iommu_domain domain; /* generic domain data structure for
408 iommu core */
99126f7c
MM		 409};
410
a647dacb
MM		 411/* PCI domain-device relationship */
412struct device_domain_info {
413 struct list_head link; /* link to domain siblings */
414 struct list_head global; /* link to global list */
276dbf99 415 u8 bus; /* PCI bus number */
a647dacb 416 u8 devfn; /* PCI devfn number */
0bcb3e28 417 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 418 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 419 struct dmar_domain *domain; /* pointer to domain */
420};
421
b94e4117
JL		 422struct dmar_rmrr_unit {
423 struct list_head list; /* list of rmrr units */
424 struct acpi_dmar_header *hdr; /* ACPI header */
425 u64 base_address; /* reserved base address*/
426 u64 end_address; /* reserved end address */
832bd858 427 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 428 int devices_cnt; /* target device count */
429};
430
431struct dmar_atsr_unit {
432 struct list_head list; /* list of ATSR units */
433 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 434 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 435 int devices_cnt; /* target device count */
436 u8 include_all:1; /* include all ports */
437};
438
439static LIST_HEAD(dmar_atsr_units);
440static LIST_HEAD(dmar_rmrr_units);
441
442#define for_each_rmrr_units(rmrr) \
443 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
444
5e0d2a6f 445static void flush_unmaps_timeout(unsigned long data);
446
b707cb02 447static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 448
80b20dd8 449#define HIGH_WATER_MARK 250
450struct deferred_flush_tables {
451 int next;
452 struct iova *iova[HIGH_WATER_MARK];
453 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 454 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 455};
456
457static struct deferred_flush_tables *deferred_flush;
458
5e0d2a6f 459/* bitmap for indexing intel_iommus */
5e0d2a6f 460static int g_num_of_iommus;
461
462static DEFINE_SPINLOCK(async_umap_flush_lock);
463static LIST_HEAD(unmaps_to_do);
464
465static int timer_on;
466static long list_size;
5e0d2a6f 467
92d03cc8 468static void domain_exit(struct dmar_domain *domain);
ba395927 469static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 470static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 471 struct device *dev);
92d03cc8 472static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 473 struct device *dev);
2a46ddf7
JL		 474static int domain_detach_iommu(struct dmar_domain *domain,
475 struct intel_iommu *iommu);
ba395927 476
d3f13810 477#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 478int dmar_disabled = 0;
479#else
480int dmar_disabled = 1;
d3f13810 481#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 482
8bc1f85c
ED		 483int intel_iommu_enabled = 0;
484EXPORT_SYMBOL_GPL(intel_iommu_enabled);
485
2d9e667e 486static int dmar_map_gfx = 1;
7d3b03ce 487static int dmar_forcedac;
5e0d2a6f 488static int intel_iommu_strict;
6dd9a7c7 489static int intel_iommu_superpage = 1;
c83b2f20
DW		 490static int intel_iommu_ecs = 1;
491
492/* We only actually use ECS when PASID support (on the new bit 40)
493 * is also advertised. Some early implementations — the ones with
494 * PASID support on bit 28 — have issues even when we *only* use
495 * extended root/context tables. */
496#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
497 ecap_pasid(iommu->ecap))
ba395927 498
c0771df8
DW		 499int intel_iommu_gfx_mapped;
500EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
501
ba395927
KA		 502#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
503static DEFINE_SPINLOCK(device_domain_lock);
504static LIST_HEAD(device_domain_list);
505
b22f6434 506static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 507
4158c2ec
JR		 508static bool translation_pre_enabled(struct intel_iommu *iommu)
509{
510 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
511}
512
091d42e4
JR		 513static void clear_translation_pre_enabled(struct intel_iommu *iommu)
514{
515 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
516}
517
4158c2ec
JR		 518static void init_translation_status(struct intel_iommu *iommu)
519{
520 u32 gsts;
521
522 gsts = readl(iommu->reg + DMAR_GSTS_REG);
523 if (gsts & DMA_GSTS_TES)
524 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
525}
526
00a77deb
JR		 527/* Convert generic 'struct iommu_domain to private struct dmar_domain */
528static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
529{
530 return container_of(dom, struct dmar_domain, domain);
531}
532
ba395927
KA		 533static int __init intel_iommu_setup(char *str)
534{
535 if (!str)
536 return -EINVAL;
537 while (*str) {
0cd5c3c8
KM		 538 if (!strncmp(str, "on", 2)) {
539 dmar_disabled = 0;
9f10e5bf 540 pr_info("IOMMU enabled\n");
0cd5c3c8 541 } else if (!strncmp(str, "off", 3)) {
ba395927 542 dmar_disabled = 1;
9f10e5bf 543 pr_info("IOMMU disabled\n");
ba395927
KA		 544 } else if (!strncmp(str, "igfx_off", 8)) {
545 dmar_map_gfx = 0;
9f10e5bf 546 pr_info("Disable GFX device mapping\n");
7d3b03ce 547 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 548 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 549 dmar_forcedac = 1;
5e0d2a6f 550 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 551 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 552 intel_iommu_strict = 1;
6dd9a7c7 553 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 554 pr_info("Disable supported super page\n");
6dd9a7c7 555 intel_iommu_superpage = 0;
c83b2f20
DW		 556 } else if (!strncmp(str, "ecs_off", 7)) {
557 printk(KERN_INFO
558 "Intel-IOMMU: disable extended context table support\n");
559 intel_iommu_ecs = 0;
ba395927
KA		 560 }
561
562 str += strcspn(str, ",");
563 while (*str == ',')
564 str++;
565 }
566 return 0;
567}
568__setup("intel_iommu=", intel_iommu_setup);
569
570static struct kmem_cache *iommu_domain_cache;
571static struct kmem_cache *iommu_devinfo_cache;
ba395927 572
9452d5bf
JR		 573static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
574{
8bf47816
JR		 575 struct dmar_domain **domains;
576 int idx = did >> 8;
577
578 domains = iommu->domains[idx];
579 if (!domains)
580 return NULL;
581
582 return domains[did & 0xff];
9452d5bf
JR		 583}
584
585static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
586 struct dmar_domain *domain)
587{
8bf47816
JR		 588 struct dmar_domain **domains;
589 int idx = did >> 8;
590
591 if (!iommu->domains[idx]) {
592 size_t size = 256 * sizeof(struct dmar_domain *);
593 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
594 }
595
596 domains = iommu->domains[idx];
597 if (WARN_ON(!domains))
598 return;
599 else
600 domains[did & 0xff] = domain;
9452d5bf
JR		 601}
602
4c923d47 603static inline void *alloc_pgtable_page(int node)
eb3fa7cb 604{
4c923d47
SS		 605 struct page *page;
606 void *vaddr = NULL;
eb3fa7cb 607
4c923d47
SS		 608 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
609 if (page)
610 vaddr = page_address(page);
eb3fa7cb 611 return vaddr;
ba395927
KA		 612}
613
614static inline void free_pgtable_page(void *vaddr)
615{
616 free_page((unsigned long)vaddr);
617}
618
619static inline void *alloc_domain_mem(void)
620{
354bb65e 621 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 622}
623
38717946 624static void free_domain_mem(void *vaddr)
ba395927
KA		 625{
626 kmem_cache_free(iommu_domain_cache, vaddr);
627}
628
629static inline void * alloc_devinfo_mem(void)
630{
354bb65e 631 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 632}
633
634static inline void free_devinfo_mem(void *vaddr)
635{
636 kmem_cache_free(iommu_devinfo_cache, vaddr);
637}
638
ab8dfe25
JL		 639static inline int domain_type_is_vm(struct dmar_domain *domain)
640{
641 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
642}
643
28ccce0d
JR		 644static inline int domain_type_is_si(struct dmar_domain *domain)
645{
646 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
647}
648
ab8dfe25
JL		 649static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
650{
651 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
652 DOMAIN_FLAG_STATIC_IDENTITY);
653}
1b573683 654
162d1b10
JL		 655static inline int domain_pfn_supported(struct dmar_domain *domain,
656 unsigned long pfn)
657{
658 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
659
660 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
661}
662
4ed0d3e6 663static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 664{
665 unsigned long sagaw;
666 int agaw = -1;
667
668 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 669 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 670 agaw >= 0; agaw--) {
671 if (test_bit(agaw, &sagaw))
672 break;
673 }
674
675 return agaw;
676}
677
4ed0d3e6
FY		 678/*
679 * Calculate max SAGAW for each iommu.
680 */
681int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
682{
683 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
684}
685
686/*
687 * calculate agaw for each iommu.
688 * "SAGAW" may be different across iommus, use a default agaw, and
689 * get a supported less agaw for iommus that don't support the default agaw.
690 */
691int iommu_calculate_agaw(struct intel_iommu *iommu)
692{
693 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
694}
695
2c2e2c38 696/* This functionin only returns single iommu in a domain */
8c11e798
WH		 697static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
698{
699 int iommu_id;
700
2c2e2c38 701 /* si_domain and vm domain should not get here. */
ab8dfe25 702 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 703 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 704 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
705 return NULL;
706
707 return g_iommus[iommu_id];
708}
709
8e604097
WH		 710static void domain_update_iommu_coherency(struct dmar_domain *domain)
711{
d0501960
DW		 712 struct dmar_drhd_unit *drhd;
713 struct intel_iommu *iommu;
2f119c78
QL		 714 bool found = false;
715 int i;
2e12bc29 716
d0501960 717 domain->iommu_coherency = 1;
8e604097 718
1b198bb0 719 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 720 found = true;
8e604097
WH		 721 if (!ecap_coherent(g_iommus[i]->ecap)) {
722 domain->iommu_coherency = 0;
723 break;
724 }
8e604097 725 }
d0501960
DW		 726 if (found)
727 return;
728
729 /* No hardware attached; use lowest common denominator */
730 rcu_read_lock();
731 for_each_active_iommu(iommu, drhd) {
732 if (!ecap_coherent(iommu->ecap)) {
733 domain->iommu_coherency = 0;
734 break;
735 }
736 }
737 rcu_read_unlock();
8e604097
WH		 738}
739
161f6934 740static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 741{
161f6934
JL		 742 struct dmar_drhd_unit *drhd;
743 struct intel_iommu *iommu;
744 int ret = 1;
58c610bd 745
161f6934
JL		 746 rcu_read_lock();
747 for_each_active_iommu(iommu, drhd) {
748 if (iommu != skip) {
749 if (!ecap_sc_support(iommu->ecap)) {
750 ret = 0;
751 break;
752 }
58c610bd 753 }
58c610bd 754 }
161f6934
JL		 755 rcu_read_unlock();
756
757 return ret;
58c610bd
SY		 758}
759
161f6934 760static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 761{
8140a95d 762 struct dmar_drhd_unit *drhd;
161f6934 763 struct intel_iommu *iommu;
8140a95d 764 int mask = 0xf;
6dd9a7c7
YS		 765
766 if (!intel_iommu_superpage) {
161f6934 767 return 0;
6dd9a7c7
YS		 768 }
769
8140a95d 770 /* set iommu_superpage to the smallest common denominator */
0e242612 771 rcu_read_lock();
8140a95d 772 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 773 if (iommu != skip) {
774 mask &= cap_super_page_val(iommu->cap);
775 if (!mask)
776 break;
6dd9a7c7
YS		 777 }
778 }
0e242612
JL		 779 rcu_read_unlock();
780
161f6934 781 return fls(mask);
6dd9a7c7
YS		 782}
783
58c610bd
SY		 784/* Some capabilities may be different across iommus */
785static void domain_update_iommu_cap(struct dmar_domain *domain)
786{
787 domain_update_iommu_coherency(domain);
161f6934
JL		 788 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
789 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 790}
791
03ecc32c
DW		 792static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
793 u8 bus, u8 devfn, int alloc)
794{
795 struct root_entry *root = &iommu->root_entry[bus];
796 struct context_entry *context;
797 u64 *entry;
798
c83b2f20 799 if (ecs_enabled(iommu)) {
03ecc32c
DW		 800 if (devfn >= 0x80) {
801 devfn -= 0x80;
802 entry = &root->hi;
803 }
804 devfn *= 2;
805 }
806 entry = &root->lo;
807 if (*entry & 1)
808 context = phys_to_virt(*entry & VTD_PAGE_MASK);
809 else {
810 unsigned long phy_addr;
811 if (!alloc)
812 return NULL;
813
814 context = alloc_pgtable_page(iommu->node);
815 if (!context)
816 return NULL;
817
818 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
819 phy_addr = virt_to_phys((void *)context);
820 *entry = phy_addr | 1;
821 __iommu_flush_cache(iommu, entry, sizeof(*entry));
822 }
823 return &context[devfn];
824}
825
4ed6a540
DW		 826static int iommu_dummy(struct device *dev)
827{
828 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
829}
830
156baca8 831static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 832{
833 struct dmar_drhd_unit *drhd = NULL;
b683b230 834 struct intel_iommu *iommu;
156baca8
DW		 835 struct device *tmp;
836 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 837 u16 segment = 0;
c7151a8d
WH		 838 int i;
839
4ed6a540
DW		 840 if (iommu_dummy(dev))
841 return NULL;
842
156baca8
DW		 843 if (dev_is_pci(dev)) {
844 pdev = to_pci_dev(dev);
845 segment = pci_domain_nr(pdev->bus);
ca5b74d2 846 } else if (has_acpi_companion(dev))
156baca8
DW		 847 dev = &ACPI_COMPANION(dev)->dev;
848
0e242612 849 rcu_read_lock();
b683b230 850 for_each_active_iommu(iommu, drhd) {
156baca8 851 if (pdev && segment != drhd->segment)
276dbf99 852 continue;
c7151a8d 853
b683b230 854 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 855 drhd->devices_cnt, i, tmp) {
856 if (tmp == dev) {
857 *bus = drhd->devices[i].bus;
858 *devfn = drhd->devices[i].devfn;
b683b230 859 goto out;
156baca8
DW		 860 }
861
862 if (!pdev || !dev_is_pci(tmp))
863 continue;
864
865 ptmp = to_pci_dev(tmp);
866 if (ptmp->subordinate &&
867 ptmp->subordinate->number <= pdev->bus->number &&
868 ptmp->subordinate->busn_res.end >= pdev->bus->number)
869 goto got_pdev;
924b6231 870 }
c7151a8d 871
156baca8
DW		 872 if (pdev && drhd->include_all) {
873 got_pdev:
874 *bus = pdev->bus->number;
875 *devfn = pdev->devfn;
b683b230 876 goto out;
156baca8 877 }
c7151a8d 878 }
b683b230 879 iommu = NULL;
156baca8 880 out:
0e242612 881 rcu_read_unlock();
c7151a8d 882
b683b230 883 return iommu;
c7151a8d
WH		 884}
885
5331fe6f
WH		 886static void domain_flush_cache(struct dmar_domain *domain,
887 void *addr, int size)
888{
889 if (!domain->iommu_coherency)
890 clflush_cache_range(addr, size);
891}
892
ba395927
KA		 893static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
894{
ba395927 895 struct context_entry *context;
03ecc32c 896 int ret = 0;
ba395927
KA		 897 unsigned long flags;
898
899 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 900 context = iommu_context_addr(iommu, bus, devfn, 0);
901 if (context)
902 ret = context_present(context);
ba395927
KA		 903 spin_unlock_irqrestore(&iommu->lock, flags);
904 return ret;
905}
906
907static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
908{
ba395927
KA		 909 struct context_entry *context;
910 unsigned long flags;
911
912 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 913 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 914 if (context) {
03ecc32c
DW		 915 context_clear_entry(context);
916 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 917 }
918 spin_unlock_irqrestore(&iommu->lock, flags);
919}
920
921static void free_context_table(struct intel_iommu *iommu)
922{
ba395927
KA		 923 int i;
924 unsigned long flags;
925 struct context_entry *context;
926
927 spin_lock_irqsave(&iommu->lock, flags);
928 if (!iommu->root_entry) {
929 goto out;
930 }
931 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 932 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 933 if (context)
934 free_pgtable_page(context);
03ecc32c 935
c83b2f20 936 if (!ecs_enabled(iommu))
03ecc32c
DW		 937 continue;
938
939 context = iommu_context_addr(iommu, i, 0x80, 0);
940 if (context)
941 free_pgtable_page(context);
942
ba395927
KA		 943 }
944 free_pgtable_page(iommu->root_entry);
945 iommu->root_entry = NULL;
946out:
947 spin_unlock_irqrestore(&iommu->lock, flags);
948}
949
b026fd28 950static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 951 unsigned long pfn, int *target_level)
ba395927 952{
ba395927
KA		 953 struct dma_pte *parent, *pte = NULL;
954 int level = agaw_to_level(domain->agaw);
4399c8bf 955 int offset;
ba395927
KA		 956
957 BUG_ON(!domain->pgd);
f9423606 958
162d1b10 959 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 960 /* Address beyond IOMMU's addressing capabilities. */
961 return NULL;
962
ba395927
KA		 963 parent = domain->pgd;
964
5cf0a76f 965 while (1) {
ba395927
KA		 966 void *tmp_page;
967
b026fd28 968 offset = pfn_level_offset(pfn, level);
ba395927 969 pte = &parent[offset];
5cf0a76f 970 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 971 break;
5cf0a76f 972 if (level == *target_level)
ba395927
KA		 973 break;
974
19c239ce 975 if (!dma_pte_present(pte)) {
c85994e4
DW		 976 uint64_t pteval;
977
4c923d47 978 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 979
206a73c1 980 if (!tmp_page)
ba395927 981 return NULL;
206a73c1 982
c85994e4 983 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 984 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 985 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 986 /* Someone else set it while we were thinking; use theirs. */
987 free_pgtable_page(tmp_page);
effad4b5 988 else
c85994e4 989 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 990 }
5cf0a76f
DW		 991 if (level == 1)
992 break;
993
19c239ce 994 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 995 level--;
996 }
997
5cf0a76f
DW		 998 if (!*target_level)
999 *target_level = level;
1000
ba395927
KA		 1001 return pte;
1002}
1003
6dd9a7c7 1004
ba395927 1005/* return address's pte at specific level */
90dcfb5e
DW		 1006static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1007 unsigned long pfn,
6dd9a7c7 1008 int level, int *large_page)
ba395927
KA		 1009{
1010 struct dma_pte *parent, *pte = NULL;
1011 int total = agaw_to_level(domain->agaw);
1012 int offset;
1013
1014 parent = domain->pgd;
1015 while (level <= total) {
90dcfb5e 1016 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1017 pte = &parent[offset];
1018 if (level == total)
1019 return pte;
1020
6dd9a7c7
YS		 1021 if (!dma_pte_present(pte)) {
1022 *large_page = total;
ba395927 1023 break;
6dd9a7c7
YS		 1024 }
1025
e16922af 1026 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1027 *large_page = total;
1028 return pte;
1029 }
1030
19c239ce 1031 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1032 total--;
1033 }
1034 return NULL;
1035}
1036
ba395927 1037/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1038static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1039 unsigned long start_pfn,
1040 unsigned long last_pfn)
ba395927 1041{
6dd9a7c7 1042 unsigned int large_page = 1;
310a5ab9 1043 struct dma_pte *first_pte, *pte;
66eae846 1044
162d1b10
JL		 1045 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1046 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1047 BUG_ON(start_pfn > last_pfn);
ba395927 1048
04b18e65 1049 /* we don't need lock here; nobody else touches the iova range */
59c36286 1050 do {
6dd9a7c7
YS		 1051 large_page = 1;
1052 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1053 if (!pte) {
6dd9a7c7 1054 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1055 continue;
1056 }
6dd9a7c7 1057 do {
310a5ab9 1058 dma_clear_pte(pte);
6dd9a7c7 1059 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1060 pte++;
75e6bf96
DW		 1061 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1062
310a5ab9
DW		 1063 domain_flush_cache(domain, first_pte,
1064 (void *)pte - (void *)first_pte);
59c36286
DW		 1065
1066 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1067}
1068
3269ee0b
AW		 1069static void dma_pte_free_level(struct dmar_domain *domain, int level,
1070 struct dma_pte *pte, unsigned long pfn,
1071 unsigned long start_pfn, unsigned long last_pfn)
1072{
1073 pfn = max(start_pfn, pfn);
1074 pte = &pte[pfn_level_offset(pfn, level)];
1075
1076 do {
1077 unsigned long level_pfn;
1078 struct dma_pte *level_pte;
1079
1080 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1081 goto next;
1082
1083 level_pfn = pfn & level_mask(level - 1);
1084 level_pte = phys_to_virt(dma_pte_addr(pte));
1085
1086 if (level > 2)
1087 dma_pte_free_level(domain, level - 1, level_pte,
1088 level_pfn, start_pfn, last_pfn);
1089
1090 /* If range covers entire pagetable, free it */
1091 if (!(start_pfn > level_pfn ||
08336fd2 1092 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1093 dma_clear_pte(pte);
1094 domain_flush_cache(domain, pte, sizeof(*pte));
1095 free_pgtable_page(level_pte);
1096 }
1097next:
1098 pfn += level_size(level);
1099 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1100}
1101
ba395927
KA		 1102/* free page table pages. last level pte should already be cleared */
1103static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1104 unsigned long start_pfn,
1105 unsigned long last_pfn)
ba395927 1106{
162d1b10
JL		 1107 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1108 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1109 BUG_ON(start_pfn > last_pfn);
ba395927 1110
d41a4adb
JL		 1111 dma_pte_clear_range(domain, start_pfn, last_pfn);
1112
f3a0a52f 1113 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1114 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1115 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1116
ba395927 1117 /* free pgd */
d794dc9b 1118 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1119 free_pgtable_page(domain->pgd);
1120 domain->pgd = NULL;
1121 }
1122}
1123
ea8ea460
DW		 1124/* When a page at a given level is being unlinked from its parent, we don't
1125 need to *modify* it at all. All we need to do is make a list of all the
1126 pages which can be freed just as soon as we've flushed the IOTLB and we
1127 know the hardware page-walk will no longer touch them.
1128 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1129 be freed. */
1130static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1131 int level, struct dma_pte *pte,
1132 struct page *freelist)
1133{
1134 struct page *pg;
1135
1136 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1137 pg->freelist = freelist;
1138 freelist = pg;
1139
1140 if (level == 1)
1141 return freelist;
1142
adeb2590
JL		 1143 pte = page_address(pg);
1144 do {
ea8ea460
DW		 1145 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1146 freelist = dma_pte_list_pagetables(domain, level - 1,
1147 pte, freelist);
adeb2590
JL		 1148 pte++;
1149 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1150
1151 return freelist;
1152}
1153
1154static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1155 struct dma_pte *pte, unsigned long pfn,
1156 unsigned long start_pfn,
1157 unsigned long last_pfn,
1158 struct page *freelist)
1159{
1160 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1161
1162 pfn = max(start_pfn, pfn);
1163 pte = &pte[pfn_level_offset(pfn, level)];
1164
1165 do {
1166 unsigned long level_pfn;
1167
1168 if (!dma_pte_present(pte))
1169 goto next;
1170
1171 level_pfn = pfn & level_mask(level);
1172
1173 /* If range covers entire pagetable, free it */
1174 if (start_pfn <= level_pfn &&
1175 last_pfn >= level_pfn + level_size(level) - 1) {
1176 /* These suborbinate page tables are going away entirely. Don't
1177 bother to clear them; we're just going to *free* them. */
1178 if (level > 1 && !dma_pte_superpage(pte))
1179 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1180
1181 dma_clear_pte(pte);
1182 if (!first_pte)
1183 first_pte = pte;
1184 last_pte = pte;
1185 } else if (level > 1) {
1186 /* Recurse down into a level that isn't *entirely* obsolete */
1187 freelist = dma_pte_clear_level(domain, level - 1,
1188 phys_to_virt(dma_pte_addr(pte)),
1189 level_pfn, start_pfn, last_pfn,
1190 freelist);
1191 }
1192next:
1193 pfn += level_size(level);
1194 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1195
1196 if (first_pte)
1197 domain_flush_cache(domain, first_pte,
1198 (void *)++last_pte - (void *)first_pte);
1199
1200 return freelist;
1201}
1202
1203/* We can't just free the pages because the IOMMU may still be walking
1204 the page tables, and may have cached the intermediate levels. The
1205 pages can only be freed after the IOTLB flush has been done. */
1206struct page *domain_unmap(struct dmar_domain *domain,
1207 unsigned long start_pfn,
1208 unsigned long last_pfn)
1209{
ea8ea460
DW		 1210 struct page *freelist = NULL;
1211
162d1b10
JL		 1212 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1213 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1214 BUG_ON(start_pfn > last_pfn);
1215
1216 /* we don't need lock here; nobody else touches the iova range */
1217 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1218 domain->pgd, 0, start_pfn, last_pfn, NULL);
1219
1220 /* free pgd */
1221 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1222 struct page *pgd_page = virt_to_page(domain->pgd);
1223 pgd_page->freelist = freelist;
1224 freelist = pgd_page;
1225
1226 domain->pgd = NULL;
1227 }
1228
1229 return freelist;
1230}
1231
1232void dma_free_pagelist(struct page *freelist)
1233{
1234 struct page *pg;
1235
1236 while ((pg = freelist)) {
1237 freelist = pg->freelist;
1238 free_pgtable_page(page_address(pg));
1239 }
1240}
1241
ba395927
KA		 1242/* iommu handling */
1243static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1244{
1245 struct root_entry *root;
1246 unsigned long flags;
1247
4c923d47 1248 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1249 if (!root) {
9f10e5bf 1250 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1251 iommu->name);
ba395927 1252 return -ENOMEM;
ffebeb46 1253 }
ba395927 1254
5b6985ce 1255 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1256
1257 spin_lock_irqsave(&iommu->lock, flags);
1258 iommu->root_entry = root;
1259 spin_unlock_irqrestore(&iommu->lock, flags);
1260
1261 return 0;
1262}
1263
ba395927
KA		 1264static void iommu_set_root_entry(struct intel_iommu *iommu)
1265{
03ecc32c 1266 u64 addr;
c416daa9 1267 u32 sts;
ba395927
KA		 1268 unsigned long flag;
1269
03ecc32c 1270 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1271 if (ecs_enabled(iommu))
03ecc32c 1272 addr |= DMA_RTADDR_RTT;
ba395927 1273
1f5b3c3f 1274 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1275 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1276
c416daa9 1277 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1278
1279 /* Make sure hardware complete it */
1280 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1281 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1282
1f5b3c3f 1283 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1284}
1285
1286static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1287{
1288 u32 val;
1289 unsigned long flag;
1290
9af88143 1291 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1292 return;
ba395927 1293
1f5b3c3f 1294 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1295 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1296
1297 /* Make sure hardware complete it */
1298 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1299 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1300
1f5b3c3f 1301 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1302}
1303
1304/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1305static void __iommu_flush_context(struct intel_iommu *iommu,
1306 u16 did, u16 source_id, u8 function_mask,
1307 u64 type)
ba395927
KA		 1308{
1309 u64 val = 0;
1310 unsigned long flag;
1311
ba395927
KA		 1312 switch (type) {
1313 case DMA_CCMD_GLOBAL_INVL:
1314 val = DMA_CCMD_GLOBAL_INVL;
1315 break;
1316 case DMA_CCMD_DOMAIN_INVL:
1317 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1318 break;
1319 case DMA_CCMD_DEVICE_INVL:
1320 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1321 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1322 break;
1323 default:
1324 BUG();
1325 }
1326 val |= DMA_CCMD_ICC;
1327
1f5b3c3f 1328 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1329 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1330
1331 /* Make sure hardware complete it */
1332 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1333 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1334
1f5b3c3f 1335 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1336}
1337
ba395927 1338/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1339static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1340 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1341{
1342 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1343 u64 val = 0, val_iva = 0;
1344 unsigned long flag;
1345
ba395927
KA		 1346 switch (type) {
1347 case DMA_TLB_GLOBAL_FLUSH:
1348 /* global flush doesn't need set IVA_REG */
1349 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1350 break;
1351 case DMA_TLB_DSI_FLUSH:
1352 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1353 break;
1354 case DMA_TLB_PSI_FLUSH:
1355 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1356 /* IH bit is passed in as part of address */
ba395927
KA		 1357 val_iva = size_order | addr;
1358 break;
1359 default:
1360 BUG();
1361 }
1362 /* Note: set drain read/write */
1363#if 0
1364 /*
1365 * This is probably to be super secure.. Looks like we can
1366 * ignore it without any impact.
1367 */
1368 if (cap_read_drain(iommu->cap))
1369 val |= DMA_TLB_READ_DRAIN;
1370#endif
1371 if (cap_write_drain(iommu->cap))
1372 val |= DMA_TLB_WRITE_DRAIN;
1373
1f5b3c3f 1374 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1375 /* Note: Only uses first TLB reg currently */
1376 if (val_iva)
1377 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1378 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1379
1380 /* Make sure hardware complete it */
1381 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1382 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1383
1f5b3c3f 1384 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1385
1386 /* check IOTLB invalidation granularity */
1387 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1388 pr_err("Flush IOTLB failed\n");
ba395927 1389 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1390 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1391 (unsigned long long)DMA_TLB_IIRG(type),
1392 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1393}
1394
64ae892b
DW		 1395static struct device_domain_info *
1396iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1397 u8 bus, u8 devfn)
93a23a72 1398{
2f119c78 1399 bool found = false;
93a23a72
YZ		 1400 unsigned long flags;
1401 struct device_domain_info *info;
0bcb3e28 1402 struct pci_dev *pdev;
93a23a72
YZ		 1403
1404 if (!ecap_dev_iotlb_support(iommu->ecap))
1405 return NULL;
1406
1407 if (!iommu->qi)
1408 return NULL;
1409
1410 spin_lock_irqsave(&device_domain_lock, flags);
1411 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1412 if (info->iommu == iommu && info->bus == bus &&
1413 info->devfn == devfn) {
2f119c78 1414 found = true;
93a23a72
YZ		 1415 break;
1416 }
1417 spin_unlock_irqrestore(&device_domain_lock, flags);
1418
0bcb3e28 1419 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1420 return NULL;
1421
0bcb3e28
DW		 1422 pdev = to_pci_dev(info->dev);
1423
1424 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1425 return NULL;
1426
0bcb3e28 1427 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1428 return NULL;
1429
93a23a72
YZ		 1430 return info;
1431}
1432
1433static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1434{
0bcb3e28 1435 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1436 return;
1437
0bcb3e28 1438 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1439}
1440
1441static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1442{
0bcb3e28
DW		 1443 if (!info->dev || !dev_is_pci(info->dev) ||
1444 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1445 return;
1446
0bcb3e28 1447 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1448}
1449
1450static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1451 u64 addr, unsigned mask)
1452{
1453 u16 sid, qdep;
1454 unsigned long flags;
1455 struct device_domain_info *info;
1456
1457 spin_lock_irqsave(&device_domain_lock, flags);
1458 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1459 struct pci_dev *pdev;
1460 if (!info->dev || !dev_is_pci(info->dev))
1461 continue;
1462
1463 pdev = to_pci_dev(info->dev);
1464 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1465 continue;
1466
1467 sid = info->bus << 8 | info->devfn;
0bcb3e28 1468 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1469 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1470 }
1471 spin_unlock_irqrestore(&device_domain_lock, flags);
1472}
1473
a1ddcbe9
JR		 1474static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1475 struct dmar_domain *domain,
1476 unsigned long pfn, unsigned int pages,
1477 int ih, int map)
ba395927 1478{
9dd2fe89 1479 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1480 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1481 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1482
ba395927
KA		 1483 BUG_ON(pages == 0);
1484
ea8ea460
DW		 1485 if (ih)
1486 ih = 1 << 6;
ba395927 1487 /*
9dd2fe89
YZ		 1488 * Fallback to domain selective flush if no PSI support or the size is
1489 * too big.
ba395927
KA		 1490 * PSI requires page size to be 2 ^ x, and the base address is naturally
1491 * aligned to the size
1492 */
9dd2fe89
YZ		 1493 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1494 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1495 DMA_TLB_DSI_FLUSH);
9dd2fe89 1496 else
ea8ea460 1497 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1498 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1499
1500 /*
82653633
NA		 1501 * In caching mode, changes of pages from non-present to present require
1502 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1503 */
82653633 1504 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1505 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1506 addr, mask);
ba395927
KA		 1507}
1508
f8bab735 1509static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1510{
1511 u32 pmen;
1512 unsigned long flags;
1513
1f5b3c3f 1514 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1515 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1516 pmen &= ~DMA_PMEN_EPM;
1517 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1518
1519 /* wait for the protected region status bit to clear */
1520 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1521 readl, !(pmen & DMA_PMEN_PRS), pmen);
1522
1f5b3c3f 1523 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1524}
1525
2a41ccee 1526static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1527{
1528 u32 sts;
1529 unsigned long flags;
1530
1f5b3c3f 1531 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1532 iommu->gcmd |= DMA_GCMD_TE;
1533 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1534
1535 /* Make sure hardware complete it */
1536 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1537 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1538
1f5b3c3f 1539 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1540}
1541
2a41ccee 1542static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1543{
1544 u32 sts;
1545 unsigned long flag;
1546
1f5b3c3f 1547 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1548 iommu->gcmd &= ~DMA_GCMD_TE;
1549 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1550
1551 /* Make sure hardware complete it */
1552 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1553 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1554
1f5b3c3f 1555 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1556}
1557
3460a6d9 1558
ba395927
KA		 1559static int iommu_init_domains(struct intel_iommu *iommu)
1560{
8bf47816
JR		 1561 u32 ndomains, nlongs;
1562 size_t size;
ba395927
KA		 1563
1564 ndomains = cap_ndoms(iommu->cap);
8bf47816 1565 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1566 iommu->name, ndomains);
ba395927
KA		 1567 nlongs = BITS_TO_LONGS(ndomains);
1568
94a91b50
DD		 1569 spin_lock_init(&iommu->lock);
1570
ba395927
KA		 1571 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1572 if (!iommu->domain_ids) {
9f10e5bf
JR		 1573 pr_err("%s: Allocating domain id array failed\n",
1574 iommu->name);
ba395927
KA		 1575 return -ENOMEM;
1576 }
8bf47816
JR		 1577
1578 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1579 iommu->domains = kzalloc(size, GFP_KERNEL);
1580
1581 if (iommu->domains) {
1582 size = 256 * sizeof(struct dmar_domain *);
1583 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1584 }
1585
1586 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1587 pr_err("%s: Allocating domain array failed\n",
1588 iommu->name);
852bdb04 1589 kfree(iommu->domain_ids);
8bf47816 1590 kfree(iommu->domains);
852bdb04 1591 iommu->domain_ids = NULL;
8bf47816 1592 iommu->domains = NULL;
ba395927
KA		 1593 return -ENOMEM;
1594 }
1595
8bf47816
JR		 1596
1597
ba395927 1598 /*
c0e8a6c8
JR		 1599 * If Caching mode is set, then invalid translations are tagged
1600 * with domain-id 0, hence we need to pre-allocate it. We also
1601 * use domain-id 0 as a marker for non-allocated domain-id, so
1602 * make sure it is not used for a real domain.
ba395927 1603 */
c0e8a6c8
JR		 1604 set_bit(0, iommu->domain_ids);
1605
ba395927
KA		 1606 return 0;
1607}
ba395927 1608
ffebeb46 1609static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1610{
1611 struct dmar_domain *domain;
2a46ddf7 1612 int i;
ba395927 1613
94a91b50 1614 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1615 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1616 /*
1617 * Domain id 0 is reserved for invalid translation
c0e8a6c8
JR		 1618 * if hardware supports caching mode and used as
1619 * a non-allocated marker.
a4eaa86c 1620 */
c0e8a6c8 1621 if (i == 0)
a4eaa86c
JL		 1622 continue;
1623
9452d5bf 1624 domain = get_iommu_domain(iommu, i);
94a91b50 1625 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1626 if (domain_detach_iommu(domain, iommu) == 0 &&
1627 !domain_type_is_vm(domain))
92d03cc8 1628 domain_exit(domain);
5e98c4b1 1629 }
ba395927
KA		 1630 }
1631
1632 if (iommu->gcmd & DMA_GCMD_TE)
1633 iommu_disable_translation(iommu);
ffebeb46 1634}
ba395927 1635
ffebeb46
JL		 1636static void free_dmar_iommu(struct intel_iommu *iommu)
1637{
1638 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1639 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1640 int i;
1641
1642 for (i = 0; i < elems; i++)
1643 kfree(iommu->domains[i]);
ffebeb46
JL		 1644 kfree(iommu->domains);
1645 kfree(iommu->domain_ids);
1646 iommu->domains = NULL;
1647 iommu->domain_ids = NULL;
1648 }
ba395927 1649
d9630fe9
WH		 1650 g_iommus[iommu->seq_id] = NULL;
1651
ba395927
KA		 1652 /* free context mapping */
1653 free_context_table(iommu);
ba395927
KA		 1654}
1655
ab8dfe25 1656static struct dmar_domain *alloc_domain(int flags)
ba395927 1657{
92d03cc8
JL		 1658 /* domain id for virtual machine, it won't be set in context */
1659 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1660 struct dmar_domain *domain;
ba395927
KA		 1661
1662 domain = alloc_domain_mem();
1663 if (!domain)
1664 return NULL;
1665
ab8dfe25 1666 memset(domain, 0, sizeof(*domain));
4c923d47 1667 domain->nid = -1;
ab8dfe25 1668 domain->flags = flags;
92d03cc8
JL		 1669 spin_lock_init(&domain->iommu_lock);
1670 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1671 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1672 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1673
1674 return domain;
1675}
1676
fb170fb4
JL		 1677static int __iommu_attach_domain(struct dmar_domain *domain,
1678 struct intel_iommu *iommu)
2c2e2c38
FY		 1679{
1680 int num;
1681 unsigned long ndomains;
2c2e2c38 1682
e2411427
JR		 1683 num = domain->iommu_did[iommu->seq_id];
1684 if (num)
1685 return num;
1686
ba395927 1687 ndomains = cap_ndoms(iommu->cap);
e2411427
JR		 1688 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1689
fb170fb4
JL		 1690 if (num < ndomains) {
1691 set_bit(num, iommu->domain_ids);
9452d5bf 1692 set_iommu_domain(iommu, num, domain);
c0e8a6c8 1693 domain->iommu_did[iommu->seq_id] = num;
fb170fb4
JL		 1694 } else {
1695 num = -ENOSPC;
ba395927
KA		 1696 }
1697
e2411427
JR		 1698 if (num < 0)
1699 pr_err("%s: No free domain ids\n", iommu->name);
1700
fb170fb4
JL		 1701 return num;
1702}
1703
1704static int iommu_attach_domain(struct dmar_domain *domain,
1705 struct intel_iommu *iommu)
1706{
1707 int num;
1708 unsigned long flags;
1709
1710 spin_lock_irqsave(&iommu->lock, flags);
1711 num = __iommu_attach_domain(domain, iommu);
44bde614 1712 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1713
fb170fb4 1714 return num;
ba395927
KA		 1715}
1716
2c2e2c38
FY		 1717static void iommu_detach_domain(struct dmar_domain *domain,
1718 struct intel_iommu *iommu)
ba395927
KA		 1719{
1720 unsigned long flags;
c0e8a6c8 1721 int num;
ba395927 1722
8c11e798 1723 spin_lock_irqsave(&iommu->lock, flags);
c0e8a6c8
JR		 1724
1725 num = domain->iommu_did[iommu->seq_id];
1726
1727 if (num == 0)
1728 return;
1729
1730 clear_bit(num, iommu->domain_ids);
9452d5bf 1731 set_iommu_domain(iommu, num, NULL);
c0e8a6c8 1732
8c11e798 1733 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1734}
1735
fb170fb4
JL		 1736static void domain_attach_iommu(struct dmar_domain *domain,
1737 struct intel_iommu *iommu)
1738{
1739 unsigned long flags;
1740
1741 spin_lock_irqsave(&domain->iommu_lock, flags);
1742 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1743 domain->iommu_count++;
1744 if (domain->iommu_count == 1)
1745 domain->nid = iommu->node;
1746 domain_update_iommu_cap(domain);
1747 }
1748 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1749}
1750
1751static int domain_detach_iommu(struct dmar_domain *domain,
1752 struct intel_iommu *iommu)
1753{
1754 unsigned long flags;
1755 int count = INT_MAX;
1756
1757 spin_lock_irqsave(&domain->iommu_lock, flags);
1758 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1759 count = --domain->iommu_count;
1760 domain_update_iommu_cap(domain);
c0e8a6c8 1761 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4
JL		 1762 }
1763 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1764
1765 return count;
1766}
1767
ba395927 1768static struct iova_domain reserved_iova_list;
8a443df4 1769static struct lock_class_key reserved_rbtree_key;
ba395927 1770
51a63e67 1771static int dmar_init_reserved_ranges(void)
ba395927
KA		 1772{
1773 struct pci_dev *pdev = NULL;
1774 struct iova *iova;
1775 int i;
ba395927 1776
0fb5fe87
RM		 1777 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1778 DMA_32BIT_PFN);
ba395927 1779
8a443df4
MG		 1780 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1781 &reserved_rbtree_key);
1782
ba395927
KA		 1783 /* IOAPIC ranges shouldn't be accessed by DMA */
1784 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1785 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1786 if (!iova) {
9f10e5bf 1787 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1788 return -ENODEV;
1789 }
ba395927
KA		 1790
1791 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1792 for_each_pci_dev(pdev) {
1793 struct resource *r;
1794
1795 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1796 r = &pdev->resource[i];
1797 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1798 continue;
1a4a4551
DW		 1799 iova = reserve_iova(&reserved_iova_list,
1800 IOVA_PFN(r->start),
1801 IOVA_PFN(r->end));
51a63e67 1802 if (!iova) {
9f10e5bf 1803 pr_err("Reserve iova failed\n");
51a63e67
JC		 1804 return -ENODEV;
1805 }
ba395927
KA		 1806 }
1807 }
51a63e67 1808 return 0;
ba395927
KA		 1809}
1810
1811static void domain_reserve_special_ranges(struct dmar_domain *domain)
1812{
1813 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1814}
1815
1816static inline int guestwidth_to_adjustwidth(int gaw)
1817{
1818 int agaw;
1819 int r = (gaw - 12) % 9;
1820
1821 if (r == 0)
1822 agaw = gaw;
1823 else
1824 agaw = gaw + 9 - r;
1825 if (agaw > 64)
1826 agaw = 64;
1827 return agaw;
1828}
1829
1830static int domain_init(struct dmar_domain *domain, int guest_width)
1831{
1832 struct intel_iommu *iommu;
1833 int adjust_width, agaw;
1834 unsigned long sagaw;
1835
0fb5fe87
RM		 1836 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1837 DMA_32BIT_PFN);
ba395927
KA		 1838 domain_reserve_special_ranges(domain);
1839
1840 /* calculate AGAW */
8c11e798 1841 iommu = domain_get_iommu(domain);
ba395927
KA		 1842 if (guest_width > cap_mgaw(iommu->cap))
1843 guest_width = cap_mgaw(iommu->cap);
1844 domain->gaw = guest_width;
1845 adjust_width = guestwidth_to_adjustwidth(guest_width);
1846 agaw = width_to_agaw(adjust_width);
1847 sagaw = cap_sagaw(iommu->cap);
1848 if (!test_bit(agaw, &sagaw)) {
1849 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1850 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1851 agaw = find_next_bit(&sagaw, 5, agaw);
1852 if (agaw >= 5)
1853 return -ENODEV;
1854 }
1855 domain->agaw = agaw;
ba395927 1856
8e604097
WH		 1857 if (ecap_coherent(iommu->ecap))
1858 domain->iommu_coherency = 1;
1859 else
1860 domain->iommu_coherency = 0;
1861
58c610bd
SY		 1862 if (ecap_sc_support(iommu->ecap))
1863 domain->iommu_snooping = 1;
1864 else
1865 domain->iommu_snooping = 0;
1866
214e39aa
DW		 1867 if (intel_iommu_superpage)
1868 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1869 else
1870 domain->iommu_superpage = 0;
1871
4c923d47 1872 domain->nid = iommu->node;
c7151a8d 1873
ba395927 1874 /* always allocate the top pgd */
4c923d47 1875 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1876 if (!domain->pgd)
1877 return -ENOMEM;
5b6985ce 1878 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1879 return 0;
1880}
1881
1882static void domain_exit(struct dmar_domain *domain)
1883{
46ebb7af
AW		 1884 struct dmar_drhd_unit *drhd;
1885 struct intel_iommu *iommu;
ea8ea460 1886 struct page *freelist = NULL;
ba395927
KA		 1887
1888 /* Domain 0 is reserved, so dont process it */
1889 if (!domain)
1890 return;
1891
7b668357
AW		 1892 /* Flush any lazy unmaps that may reference this domain */
1893 if (!intel_iommu_strict)
1894 flush_unmaps_timeout(0);
1895
92d03cc8 1896 /* remove associated devices */
ba395927 1897 domain_remove_dev_info(domain);
92d03cc8 1898
ba395927
KA		 1899 /* destroy iovas */
1900 put_iova_domain(&domain->iovad);
ba395927 1901
ea8ea460 1902 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1903
92d03cc8 1904 /* clear attached or cached domains */
0e242612 1905 rcu_read_lock();
46ebb7af
AW		 1906 for_each_active_iommu(iommu, drhd)
1907 if (domain_type_is_vm(domain) ||
1908 test_bit(iommu->seq_id, domain->iommu_bmp))
1909 iommu_detach_domain(domain, iommu);
0e242612 1910 rcu_read_unlock();
2c2e2c38 1911
ea8ea460
DW		 1912 dma_free_pagelist(freelist);
1913
ba395927
KA		 1914 free_domain_mem(domain);
1915}
1916
64ae892b
DW		 1917static int domain_context_mapping_one(struct dmar_domain *domain,
1918 struct intel_iommu *iommu,
28ccce0d 1919 u8 bus, u8 devfn)
ba395927 1920{
28ccce0d
JR		 1921 int translation = CONTEXT_TT_MULTI_LEVEL;
1922 struct device_domain_info *info = NULL;
ba395927 1923 struct context_entry *context;
ba395927 1924 unsigned long flags;
ea6606b0 1925 struct dma_pte *pgd;
ea6606b0
WH		 1926 int id;
1927 int agaw;
28ccce0d
JR		 1928
1929 if (hw_pass_through && domain_type_is_si(domain))
1930 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1931
1932 pr_debug("Set context mapping for %02x:%02x.%d\n",
1933 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1934
ba395927 1935 BUG_ON(!domain->pgd);
5331fe6f 1936
03ecc32c
DW		 1937 spin_lock_irqsave(&iommu->lock, flags);
1938 context = iommu_context_addr(iommu, bus, devfn, 1);
1939 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1940 if (!context)
1941 return -ENOMEM;
1942 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1943 if (context_present(context)) {
ba395927
KA		 1944 spin_unlock_irqrestore(&iommu->lock, flags);
1945 return 0;
1946 }
1947
ea6606b0
WH		 1948 pgd = domain->pgd;
1949
de24e553
JR		 1950 id = __iommu_attach_domain(domain, iommu);
1951 if (id < 0) {
1952 spin_unlock_irqrestore(&iommu->lock, flags);
1953 pr_err("%s: No free domain ids\n", iommu->name);
1954 return -EFAULT;
ea6606b0
WH		 1955 }
1956
de24e553 1957 context_clear_entry(context);
ea6606b0 1958 context_set_domain_id(context, id);
4ed0d3e6 1959
de24e553
JR		 1960 /*
1961 * Skip top levels of page tables for iommu which has less agaw
1962 * than default. Unnecessary for PT mode.
1963 */
93a23a72 1964 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553
JR		 1965 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1966 pgd = phys_to_virt(dma_pte_addr(pgd));
1967 if (!dma_pte_present(pgd)) {
1968 spin_unlock_irqrestore(&iommu->lock, flags);
1969 return -ENOMEM;
1970 }
1971 }
1972
64ae892b 1973 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1974 translation = info ? CONTEXT_TT_DEV_IOTLB :
1975 CONTEXT_TT_MULTI_LEVEL;
de24e553 1976
93a23a72
YZ		 1977 context_set_address_root(context, virt_to_phys(pgd));
1978 context_set_address_width(context, iommu->agaw);
de24e553
JR		 1979 } else {
1980 /*
1981 * In pass through mode, AW must be programmed to
1982 * indicate the largest AGAW value supported by
1983 * hardware. And ASR is ignored by hardware.
1984 */
1985 context_set_address_width(context, iommu->msagaw);
93a23a72 1986 }
4ed0d3e6
FY		 1987
1988 context_set_translation_type(context, translation);
c07e7d21
MM		 1989 context_set_fault_enable(context);
1990 context_set_present(context);
5331fe6f 1991 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1992
4c25a2c1
DW		 1993 /*
1994 * It's a non-present to present mapping. If hardware doesn't cache
1995 * non-present entry we only need to flush the write-buffer. If the
1996 * _does_ cache non-present entries, then it does so in the special
1997 * domain #0, which we have to flush:
1998 */
1999 if (cap_caching_mode(iommu->cap)) {
2000 iommu->flush.flush_context(iommu, 0,
2001 (((u16)bus) << 8) | devfn,
2002 DMA_CCMD_MASK_NOBIT,
2003 DMA_CCMD_DEVICE_INVL);
18fd779a 2004 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 2005 } else {
ba395927 2006 iommu_flush_write_buffer(iommu);
4c25a2c1 2007 }
93a23a72 2008 iommu_enable_dev_iotlb(info);
ba395927 2009 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 2010
fb170fb4
JL		 2011 domain_attach_iommu(domain, iommu);
2012
ba395927
KA		 2013 return 0;
2014}
2015
579305f7
AW		 2016struct domain_context_mapping_data {
2017 struct dmar_domain *domain;
2018 struct intel_iommu *iommu;
579305f7
AW		 2019};
2020
2021static int domain_context_mapping_cb(struct pci_dev *pdev,
2022 u16 alias, void *opaque)
2023{
2024 struct domain_context_mapping_data *data = opaque;
2025
2026 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2027 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2028}
2029
ba395927 2030static int
28ccce0d 2031domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2032{
64ae892b 2033 struct intel_iommu *iommu;
156baca8 2034 u8 bus, devfn;
579305f7 2035 struct domain_context_mapping_data data;
64ae892b 2036
e1f167f3 2037 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2038 if (!iommu)
2039 return -ENODEV;
ba395927 2040
579305f7 2041 if (!dev_is_pci(dev))
28ccce0d 2042 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2043
2044 data.domain = domain;
2045 data.iommu = iommu;
579305f7
AW		 2046
2047 return pci_for_each_dma_alias(to_pci_dev(dev),
2048 &domain_context_mapping_cb, &data);
2049}
2050
2051static int domain_context_mapped_cb(struct pci_dev *pdev,
2052 u16 alias, void *opaque)
2053{
2054 struct intel_iommu *iommu = opaque;
2055
2056 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2057}
2058
e1f167f3 2059static int domain_context_mapped(struct device *dev)
ba395927 2060{
5331fe6f 2061 struct intel_iommu *iommu;
156baca8 2062 u8 bus, devfn;
5331fe6f 2063
e1f167f3 2064 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2065 if (!iommu)
2066 return -ENODEV;
ba395927 2067
579305f7
AW		 2068 if (!dev_is_pci(dev))
2069 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2070
579305f7
AW		 2071 return !pci_for_each_dma_alias(to_pci_dev(dev),
2072 domain_context_mapped_cb, iommu);
ba395927
KA		 2073}
2074
f532959b
FY		 2075/* Returns a number of VTD pages, but aligned to MM page size */
2076static inline unsigned long aligned_nrpages(unsigned long host_addr,
2077 size_t size)
2078{
2079 host_addr &= ~PAGE_MASK;
2080 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2081}
2082
6dd9a7c7
YS		 2083/* Return largest possible superpage level for a given mapping */
2084static inline int hardware_largepage_caps(struct dmar_domain *domain,
2085 unsigned long iov_pfn,
2086 unsigned long phy_pfn,
2087 unsigned long pages)
2088{
2089 int support, level = 1;
2090 unsigned long pfnmerge;
2091
2092 support = domain->iommu_superpage;
2093
2094 /* To use a large page, the virtual *and* physical addresses
2095 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2096 of them will mean we have to use smaller pages. So just
2097 merge them and check both at once. */
2098 pfnmerge = iov_pfn | phy_pfn;
2099
2100 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2101 pages >>= VTD_STRIDE_SHIFT;
2102 if (!pages)
2103 break;
2104 pfnmerge >>= VTD_STRIDE_SHIFT;
2105 level++;
2106 support--;
2107 }
2108 return level;
2109}
2110
9051aa02
DW		 2111static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2112 struct scatterlist *sg, unsigned long phys_pfn,
2113 unsigned long nr_pages, int prot)
e1605495
DW		 2114{
2115 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2116 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2117 unsigned long sg_res = 0;
6dd9a7c7
YS		 2118 unsigned int largepage_lvl = 0;
2119 unsigned long lvl_pages = 0;
e1605495 2120
162d1b10 2121 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2122
2123 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2124 return -EINVAL;
2125
2126 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2127
cc4f14aa
JL		 2128 if (!sg) {
2129 sg_res = nr_pages;
9051aa02
DW		 2130 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2131 }
2132
6dd9a7c7 2133 while (nr_pages > 0) {
c85994e4
DW		 2134 uint64_t tmp;
2135
e1605495 2136 if (!sg_res) {
f532959b 2137 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2138 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2139 sg->dma_length = sg->length;
2140 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2141 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2142 }
6dd9a7c7 2143
e1605495 2144 if (!pte) {
6dd9a7c7
YS		 2145 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2146
5cf0a76f 2147 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2148 if (!pte)
2149 return -ENOMEM;
6dd9a7c7 2150 /* It is large page*/
6491d4d0 2151 if (largepage_lvl > 1) {
6dd9a7c7 2152 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2153 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2154 /*
2155 * Ensure that old small page tables are
2156 * removed to make room for superpage,
2157 * if they exist.
2158 */
6491d4d0 2159 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2160 iov_pfn + lvl_pages - 1);
6491d4d0 2161 } else {
6dd9a7c7 2162 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2163 }
6dd9a7c7 2164
e1605495
DW		 2165 }
2166 /* We don't need lock here, nobody else
2167 * touches the iova range
2168 */
7766a3fb 2169 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2170 if (tmp) {
1bf20f0d 2171 static int dumps = 5;
9f10e5bf
JR		 2172 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2173 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2174 if (dumps) {
2175 dumps--;
2176 debug_dma_dump_mappings(NULL);
2177 }
2178 WARN_ON(1);
2179 }
6dd9a7c7
YS		 2180
2181 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2182
2183 BUG_ON(nr_pages < lvl_pages);
2184 BUG_ON(sg_res < lvl_pages);
2185
2186 nr_pages -= lvl_pages;
2187 iov_pfn += lvl_pages;
2188 phys_pfn += lvl_pages;
2189 pteval += lvl_pages * VTD_PAGE_SIZE;
2190 sg_res -= lvl_pages;
2191
2192 /* If the next PTE would be the first in a new page, then we
2193 need to flush the cache on the entries we've just written.
2194 And then we'll need to recalculate 'pte', so clear it and
2195 let it get set again in the if (!pte) block above.
2196
2197 If we're done (!nr_pages) we need to flush the cache too.
2198
2199 Also if we've been setting superpages, we may need to
2200 recalculate 'pte' and switch back to smaller pages for the
2201 end of the mapping, if the trailing size is not enough to
2202 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2203 pte++;
6dd9a7c7
YS		 2204 if (!nr_pages || first_pte_in_page(pte) ||
2205 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2206 domain_flush_cache(domain, first_pte,
2207 (void *)pte - (void *)first_pte);
2208 pte = NULL;
2209 }
6dd9a7c7
YS		 2210
2211 if (!sg_res && nr_pages)
e1605495
DW		 2212 sg = sg_next(sg);
2213 }
2214 return 0;
2215}
2216
9051aa02
DW		 2217static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2218 struct scatterlist *sg, unsigned long nr_pages,
2219 int prot)
ba395927 2220{
9051aa02
DW		 2221 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2222}
6f6a00e4 2223
9051aa02
DW		 2224static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2225 unsigned long phys_pfn, unsigned long nr_pages,
2226 int prot)
2227{
2228 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2229}
2230
c7151a8d 2231static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2232{
c7151a8d
WH		 2233 if (!iommu)
2234 return;
8c11e798
WH		 2235
2236 clear_context_table(iommu, bus, devfn);
2237 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2238 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2239 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2240}
2241
109b9b04
DW		 2242static inline void unlink_domain_info(struct device_domain_info *info)
2243{
2244 assert_spin_locked(&device_domain_lock);
2245 list_del(&info->link);
2246 list_del(&info->global);
2247 if (info->dev)
0bcb3e28 2248 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2249}
2250
ba395927
KA		 2251static void domain_remove_dev_info(struct dmar_domain *domain)
2252{
3a74ca01 2253 struct device_domain_info *info, *tmp;
fb170fb4 2254 unsigned long flags;
ba395927
KA		 2255
2256 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2257 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2258 unlink_domain_info(info);
ba395927
KA		 2259 spin_unlock_irqrestore(&device_domain_lock, flags);
2260
93a23a72 2261 iommu_disable_dev_iotlb(info);
7c7faa11 2262 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2263
ab8dfe25 2264 if (domain_type_is_vm(domain)) {
7c7faa11 2265 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2266 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2267 }
2268
2269 free_devinfo_mem(info);
ba395927
KA		 2270 spin_lock_irqsave(&device_domain_lock, flags);
2271 }
2272 spin_unlock_irqrestore(&device_domain_lock, flags);
2273}
2274
2275/*
2276 * find_domain
1525a29a 2277 * Note: we use struct device->archdata.iommu stores the info
ba395927 2278 */
1525a29a 2279static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2280{
2281 struct device_domain_info *info;
2282
2283 /* No lock here, assumes no domain exit in normal case */
1525a29a 2284 info = dev->archdata.iommu;
ba395927
KA		 2285 if (info)
2286 return info->domain;
2287 return NULL;
2288}
2289
5a8f40e8 2290static inline struct device_domain_info *
745f2586
JL		 2291dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2292{
2293 struct device_domain_info *info;
2294
2295 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2296 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2297 info->devfn == devfn)
5a8f40e8 2298 return info;
745f2586
JL		 2299
2300 return NULL;
2301}
2302
5a8f40e8 2303static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2304 int bus, int devfn,
b718cd3d
DW		 2305 struct device *dev,
2306 struct dmar_domain *domain)
745f2586 2307{
5a8f40e8 2308 struct dmar_domain *found = NULL;
745f2586
JL		 2309 struct device_domain_info *info;
2310 unsigned long flags;
2311
2312 info = alloc_devinfo_mem();
2313 if (!info)
b718cd3d 2314 return NULL;
745f2586 2315
745f2586
JL		 2316 info->bus = bus;
2317 info->devfn = devfn;
2318 info->dev = dev;
2319 info->domain = domain;
5a8f40e8 2320 info->iommu = iommu;
745f2586
JL		 2321
2322 spin_lock_irqsave(&device_domain_lock, flags);
2323 if (dev)
0bcb3e28 2324 found = find_domain(dev);
5a8f40e8
DW		 2325 else {
2326 struct device_domain_info *info2;
41e80dca 2327 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2328 if (info2)
2329 found = info2->domain;
2330 }
745f2586
JL		 2331 if (found) {
2332 spin_unlock_irqrestore(&device_domain_lock, flags);
2333 free_devinfo_mem(info);
b718cd3d
DW		 2334 /* Caller must free the original domain */
2335 return found;
745f2586
JL		 2336 }
2337
b718cd3d
DW		 2338 list_add(&info->link, &domain->devices);
2339 list_add(&info->global, &device_domain_list);
2340 if (dev)
2341 dev->archdata.iommu = info;
2342 spin_unlock_irqrestore(&device_domain_lock, flags);
2343
2344 return domain;
745f2586
JL		 2345}
2346
579305f7
AW		 2347static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2348{
2349 *(u16 *)opaque = alias;
2350 return 0;
2351}
2352
ba395927 2353/* domain is initialized */
146922ec 2354static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2355{
579305f7
AW		 2356 struct dmar_domain *domain, *tmp;
2357 struct intel_iommu *iommu;
5a8f40e8 2358 struct device_domain_info *info;
579305f7 2359 u16 dma_alias;
ba395927 2360 unsigned long flags;
aa4d066a 2361 u8 bus, devfn;
ba395927 2362
146922ec 2363 domain = find_domain(dev);
ba395927
KA		 2364 if (domain)
2365 return domain;
2366
579305f7
AW		 2367 iommu = device_to_iommu(dev, &bus, &devfn);
2368 if (!iommu)
2369 return NULL;
2370
146922ec
DW		 2371 if (dev_is_pci(dev)) {
2372 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2373
579305f7
AW		 2374 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2375
2376 spin_lock_irqsave(&device_domain_lock, flags);
2377 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2378 PCI_BUS_NUM(dma_alias),
2379 dma_alias & 0xff);
2380 if (info) {
2381 iommu = info->iommu;
2382 domain = info->domain;
5a8f40e8 2383 }
579305f7 2384 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2385
579305f7
AW		 2386 /* DMA alias already has a domain, uses it */
2387 if (info)
2388 goto found_domain;
2389 }
ba395927 2390
146922ec 2391 /* Allocate and initialize new domain for the device */
ab8dfe25 2392 domain = alloc_domain(0);
745f2586 2393 if (!domain)
579305f7 2394 return NULL;
44bde614
JL		 2395 domain->id = iommu_attach_domain(domain, iommu);
2396 if (domain->id < 0) {
2fe9723d 2397 free_domain_mem(domain);
579305f7 2398 return NULL;
2c2e2c38 2399 }
fb170fb4 2400 domain_attach_iommu(domain, iommu);
579305f7
AW		 2401 if (domain_init(domain, gaw)) {
2402 domain_exit(domain);
2403 return NULL;
2c2e2c38 2404 }
ba395927 2405
579305f7
AW		 2406 /* register PCI DMA alias device */
2407 if (dev_is_pci(dev)) {
2408 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2409 dma_alias & 0xff, NULL, domain);
2410
2411 if (!tmp || tmp != domain) {
2412 domain_exit(domain);
2413 domain = tmp;
2414 }
2415
b718cd3d 2416 if (!domain)
579305f7 2417 return NULL;
ba395927
KA		 2418 }
2419
2420found_domain:
579305f7
AW		 2421 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2422
2423 if (!tmp || tmp != domain) {
2424 domain_exit(domain);
2425 domain = tmp;
2426 }
b718cd3d
DW		 2427
2428 return domain;
ba395927
KA		 2429}
2430
2c2e2c38 2431static int iommu_identity_mapping;
e0fc7e0b
DW		 2432#define IDENTMAP_ALL 1
2433#define IDENTMAP_GFX 2
2434#define IDENTMAP_AZALIA 4
2c2e2c38 2435
b213203e
DW		 2436static int iommu_domain_identity_map(struct dmar_domain *domain,
2437 unsigned long long start,
2438 unsigned long long end)
ba395927 2439{
c5395d5c
DW		 2440 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2441 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2442
2443 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2444 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2445 pr_err("Reserving iova failed\n");
b213203e 2446 return -ENOMEM;
ba395927
KA		 2447 }
2448
c5395d5c
DW		 2449 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2450 start, end, domain->id);
ba395927
KA		 2451 /*
2452 * RMRR range might have overlap with physical memory range,
2453 * clear it first
2454 */
c5395d5c 2455 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2456
c5395d5c
DW		 2457 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2458 last_vpfn - first_vpfn + 1,
61df7443 2459 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2460}
2461
0b9d9753 2462static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2463 unsigned long long start,
2464 unsigned long long end)
2465{
2466 struct dmar_domain *domain;
2467 int ret;
2468
0b9d9753 2469 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2470 if (!domain)
2471 return -ENOMEM;
2472
19943b0e
DW		 2473 /* For _hardware_ passthrough, don't bother. But for software
2474 passthrough, we do it anyway -- it may indicate a memory
2475 range which is reserved in E820, so which didn't get set
2476 up to start with in si_domain */
2477 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2478 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2479 dev_name(dev), start, end);
19943b0e
DW		 2480 return 0;
2481 }
2482
9f10e5bf
JR		 2483 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2484 dev_name(dev), start, end);
2485
5595b528
DW		 2486 if (end < start) {
2487 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2488 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2489 dmi_get_system_info(DMI_BIOS_VENDOR),
2490 dmi_get_system_info(DMI_BIOS_VERSION),
2491 dmi_get_system_info(DMI_PRODUCT_VERSION));
2492 ret = -EIO;
2493 goto error;
2494 }
2495
2ff729f5
DW		 2496 if (end >> agaw_to_width(domain->agaw)) {
2497 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2498 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2499 agaw_to_width(domain->agaw),
2500 dmi_get_system_info(DMI_BIOS_VENDOR),
2501 dmi_get_system_info(DMI_BIOS_VERSION),
2502 dmi_get_system_info(DMI_PRODUCT_VERSION));
2503 ret = -EIO;
2504 goto error;
2505 }
19943b0e 2506
b213203e 2507 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2508 if (ret)
2509 goto error;
2510
2511 /* context entry init */
28ccce0d 2512 ret = domain_context_mapping(domain, dev);
b213203e
DW		 2513 if (ret)
2514 goto error;
2515
2516 return 0;
2517
2518 error:
ba395927
KA		 2519 domain_exit(domain);
2520 return ret;
ba395927
KA		 2521}
2522
2523static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2524 struct device *dev)
ba395927 2525{
0b9d9753 2526 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2527 return 0;
0b9d9753
DW		 2528 return iommu_prepare_identity_map(dev, rmrr->base_address,
2529 rmrr->end_address);
ba395927
KA		 2530}
2531
d3f13810 2532#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2533static inline void iommu_prepare_isa(void)
2534{
2535 struct pci_dev *pdev;
2536 int ret;
2537
2538 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2539 if (!pdev)
2540 return;
2541
9f10e5bf 2542 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2543 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2544
2545 if (ret)
9f10e5bf 2546 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2547
9b27e82d 2548 pci_dev_put(pdev);
49a0429e
KA		 2549}
2550#else
2551static inline void iommu_prepare_isa(void)
2552{
2553 return;
2554}
d3f13810 2555#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2556
2c2e2c38 2557static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2558
071e1374 2559static int __init si_domain_init(int hw)
2c2e2c38 2560{
c7ab48d2 2561 int nid, ret = 0;
2c2e2c38 2562
ab8dfe25 2563 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2564 if (!si_domain)
2565 return -EFAULT;
2566
2c2e2c38
FY		 2567 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2568 domain_exit(si_domain);
2569 return -EFAULT;
2570 }
2571
0dc79715 2572 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2573
19943b0e
DW		 2574 if (hw)
2575 return 0;
2576
c7ab48d2 2577 for_each_online_node(nid) {
5dfe8660
TH		 2578 unsigned long start_pfn, end_pfn;
2579 int i;
2580
2581 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2582 ret = iommu_domain_identity_map(si_domain,
2583 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2584 if (ret)
2585 return ret;
2586 }
c7ab48d2
DW		 2587 }
2588
2c2e2c38
FY		 2589 return 0;
2590}
2591
9b226624 2592static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2593{
2594 struct device_domain_info *info;
2595
2596 if (likely(!iommu_identity_mapping))
2597 return 0;
2598
9b226624 2599 info = dev->archdata.iommu;
cb452a40
MT		 2600 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2601 return (info->domain == si_domain);
2c2e2c38 2602
2c2e2c38
FY		 2603 return 0;
2604}
2605
28ccce0d 2606static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2607{
0ac72664 2608 struct dmar_domain *ndomain;
5a8f40e8 2609 struct intel_iommu *iommu;
156baca8 2610 u8 bus, devfn;
5fe60f4e 2611 int ret;
2c2e2c38 2612
5913c9bf 2613 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2614 if (!iommu)
2615 return -ENODEV;
2616
5913c9bf 2617 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2618 if (ndomain != domain)
2619 return -EBUSY;
2c2e2c38 2620
28ccce0d 2621 ret = domain_context_mapping(domain, dev);
e2ad23d0 2622 if (ret) {
5913c9bf 2623 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2624 return ret;
2625 }
2626
2c2e2c38
FY		 2627 return 0;
2628}
2629
0b9d9753 2630static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2631{
2632 struct dmar_rmrr_unit *rmrr;
832bd858 2633 struct device *tmp;
ea2447f7
TM		 2634 int i;
2635
0e242612 2636 rcu_read_lock();
ea2447f7 2637 for_each_rmrr_units(rmrr) {
b683b230
JL		 2638 /*
2639 * Return TRUE if this RMRR contains the device that
2640 * is passed in.
2641 */
2642 for_each_active_dev_scope(rmrr->devices,
2643 rmrr->devices_cnt, i, tmp)
0b9d9753 2644 if (tmp == dev) {
0e242612 2645 rcu_read_unlock();
ea2447f7 2646 return true;
b683b230 2647 }
ea2447f7 2648 }
0e242612 2649 rcu_read_unlock();
ea2447f7
TM		 2650 return false;
2651}
2652
c875d2c1
AW		 2653/*
2654 * There are a couple cases where we need to restrict the functionality of
2655 * devices associated with RMRRs. The first is when evaluating a device for
2656 * identity mapping because problems exist when devices are moved in and out
2657 * of domains and their respective RMRR information is lost. This means that
2658 * a device with associated RMRRs will never be in a "passthrough" domain.
2659 * The second is use of the device through the IOMMU API. This interface
2660 * expects to have full control of the IOVA space for the device. We cannot
2661 * satisfy both the requirement that RMRR access is maintained and have an
2662 * unencumbered IOVA space. We also have no ability to quiesce the device's
2663 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2664 * We therefore prevent devices associated with an RMRR from participating in
2665 * the IOMMU API, which eliminates them from device assignment.
2666 *
2667 * In both cases we assume that PCI USB devices with RMRRs have them largely
2668 * for historical reasons and that the RMRR space is not actively used post
2669 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2670 *
2671 * The same exception is made for graphics devices, with the requirement that
2672 * any use of the RMRR regions will be torn down before assigning the device
2673 * to a guest.
c875d2c1
AW		 2674 */
2675static bool device_is_rmrr_locked(struct device *dev)
2676{
2677 if (!device_has_rmrr(dev))
2678 return false;
2679
2680 if (dev_is_pci(dev)) {
2681 struct pci_dev *pdev = to_pci_dev(dev);
2682
18436afd 2683 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2684 return false;
2685 }
2686
2687 return true;
2688}
2689
3bdb2591 2690static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2691{
ea2447f7 2692
3bdb2591
DW		 2693 if (dev_is_pci(dev)) {
2694 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2695
c875d2c1 2696 if (device_is_rmrr_locked(dev))
3bdb2591 2697 return 0;
e0fc7e0b 2698
3bdb2591
DW		 2699 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2700 return 1;
e0fc7e0b 2701
3bdb2591
DW		 2702 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2703 return 1;
6941af28 2704
3bdb2591 2705 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2706 return 0;
3bdb2591
DW		 2707
2708 /*
2709 * We want to start off with all devices in the 1:1 domain, and
2710 * take them out later if we find they can't access all of memory.
2711 *
2712 * However, we can't do this for PCI devices behind bridges,
2713 * because all PCI devices behind the same bridge will end up
2714 * with the same source-id on their transactions.
2715 *
2716 * Practically speaking, we can't change things around for these
2717 * devices at run-time, because we can't be sure there'll be no
2718 * DMA transactions in flight for any of their siblings.
2719 *
2720 * So PCI devices (unless they're on the root bus) as well as
2721 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2722 * the 1:1 domain, just in _case_ one of their siblings turns out
2723 * not to be able to map all of memory.
2724 */
2725 if (!pci_is_pcie(pdev)) {
2726 if (!pci_is_root_bus(pdev->bus))
2727 return 0;
2728 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2729 return 0;
2730 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2731 return 0;
3bdb2591
DW		 2732 } else {
2733 if (device_has_rmrr(dev))
2734 return 0;
2735 }
3dfc813d 2736
3bdb2591 2737 /*
3dfc813d 2738 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2739 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2740 * take them out of the 1:1 domain later.
2741 */
8fcc5372
CW		 2742 if (!startup) {
2743 /*
2744 * If the device's dma_mask is less than the system's memory
2745 * size then this is not a candidate for identity mapping.
2746 */
3bdb2591 2747 u64 dma_mask = *dev->dma_mask;
8fcc5372 2748
3bdb2591
DW		 2749 if (dev->coherent_dma_mask &&
2750 dev->coherent_dma_mask < dma_mask)
2751 dma_mask = dev->coherent_dma_mask;
8fcc5372 2752
3bdb2591 2753 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2754 }
6941af28
DW		 2755
2756 return 1;
2757}
2758
cf04eee8
DW		 2759static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2760{
2761 int ret;
2762
2763 if (!iommu_should_identity_map(dev, 1))
2764 return 0;
2765
28ccce0d 2766 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2767 if (!ret)
9f10e5bf
JR		 2768 pr_info("%s identity mapping for device %s\n",
2769 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2770 else if (ret == -ENODEV)
2771 /* device not associated with an iommu */
2772 ret = 0;
2773
2774 return ret;
2775}
2776
2777
071e1374 2778static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2779{
2c2e2c38 2780 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2781 struct dmar_drhd_unit *drhd;
2782 struct intel_iommu *iommu;
2783 struct device *dev;
2784 int i;
2785 int ret = 0;
2c2e2c38 2786
2c2e2c38 2787 for_each_pci_dev(pdev) {
cf04eee8
DW		 2788 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2789 if (ret)
2790 return ret;
2791 }
2792
2793 for_each_active_iommu(iommu, drhd)
2794 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2795 struct acpi_device_physical_node *pn;
2796 struct acpi_device *adev;
2797
2798 if (dev->bus != &acpi_bus_type)
2799 continue;
86080ccc 2800
cf04eee8
DW		 2801 adev= to_acpi_device(dev);
2802 mutex_lock(&adev->physical_node_lock);
2803 list_for_each_entry(pn, &adev->physical_node_list, node) {
2804 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2805 if (ret)
2806 break;
eae460b6 2807 }
cf04eee8
DW		 2808 mutex_unlock(&adev->physical_node_lock);
2809 if (ret)
2810 return ret;
62edf5dc 2811 }
2c2e2c38
FY		 2812
2813 return 0;
2814}
2815
ffebeb46
JL		 2816static void intel_iommu_init_qi(struct intel_iommu *iommu)
2817{
2818 /*
2819 * Start from the sane iommu hardware state.
2820 * If the queued invalidation is already initialized by us
2821 * (for example, while enabling interrupt-remapping) then
2822 * we got the things already rolling from a sane state.
2823 */
2824 if (!iommu->qi) {
2825 /*
2826 * Clear any previous faults.
2827 */
2828 dmar_fault(-1, iommu);
2829 /*
2830 * Disable queued invalidation if supported and already enabled
2831 * before OS handover.
2832 */
2833 dmar_disable_qi(iommu);
2834 }
2835
2836 if (dmar_enable_qi(iommu)) {
2837 /*
2838 * Queued Invalidate not enabled, use Register Based Invalidate
2839 */
2840 iommu->flush.flush_context = __iommu_flush_context;
2841 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2842 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2843 iommu->name);
2844 } else {
2845 iommu->flush.flush_context = qi_flush_context;
2846 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2847 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2848 }
2849}
2850
091d42e4
JR		 2851static int copy_context_table(struct intel_iommu *iommu,
2852 struct root_entry *old_re,
2853 struct context_entry **tbl,
2854 int bus, bool ext)
2855{
2856 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2857 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2858 phys_addr_t old_ce_phys;
2859
2860 tbl_idx = ext ? bus * 2 : bus;
2861
2862 for (devfn = 0; devfn < 256; devfn++) {
2863 /* First calculate the correct index */
2864 idx = (ext ? devfn * 2 : devfn) % 256;
2865
2866 if (idx == 0) {
2867 /* First save what we may have and clean up */
2868 if (new_ce) {
2869 tbl[tbl_idx] = new_ce;
2870 __iommu_flush_cache(iommu, new_ce,
2871 VTD_PAGE_SIZE);
2872 pos = 1;
2873 }
2874
2875 if (old_ce)
2876 iounmap(old_ce);
2877
2878 ret = 0;
2879 if (devfn < 0x80)
2880 old_ce_phys = root_entry_lctp(old_re);
2881 else
2882 old_ce_phys = root_entry_uctp(old_re);
2883
2884 if (!old_ce_phys) {
2885 if (ext && devfn == 0) {
2886 /* No LCTP, try UCTP */
2887 devfn = 0x7f;
2888 continue;
2889 } else {
2890 goto out;
2891 }
2892 }
2893
2894 ret = -ENOMEM;
2895 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2896 if (!old_ce)
2897 goto out;
2898
2899 new_ce = alloc_pgtable_page(iommu->node);
2900 if (!new_ce)
2901 goto out_unmap;
2902
2903 ret = 0;
2904 }
2905
2906 /* Now copy the context entry */
2907 ce = old_ce[idx];
2908
cf484d0e 2909 if (!__context_present(&ce))
091d42e4
JR		 2910 continue;
2911
dbcd861f
JR		 2912 did = context_domain_id(&ce);
2913 if (did >= 0 && did < cap_ndoms(iommu->cap))
2914 set_bit(did, iommu->domain_ids);
2915
cf484d0e
JR		 2916 /*
2917 * We need a marker for copied context entries. This
2918 * marker needs to work for the old format as well as
2919 * for extended context entries.
2920 *
2921 * Bit 67 of the context entry is used. In the old
2922 * format this bit is available to software, in the
2923 * extended format it is the PGE bit, but PGE is ignored
2924 * by HW if PASIDs are disabled (and thus still
2925 * available).
2926 *
2927 * So disable PASIDs first and then mark the entry
2928 * copied. This means that we don't copy PASID
2929 * translations from the old kernel, but this is fine as
2930 * faults there are not fatal.
2931 */
2932 context_clear_pasid_enable(&ce);
2933 context_set_copied(&ce);
2934
091d42e4
JR		 2935 new_ce[idx] = ce;
2936 }
2937
2938 tbl[tbl_idx + pos] = new_ce;
2939
2940 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2941
2942out_unmap:
2943 iounmap(old_ce);
2944
2945out:
2946 return ret;
2947}
2948
2949static int copy_translation_tables(struct intel_iommu *iommu)
2950{
2951 struct context_entry **ctxt_tbls;
2952 struct root_entry *old_rt;
2953 phys_addr_t old_rt_phys;
2954 int ctxt_table_entries;
2955 unsigned long flags;
2956 u64 rtaddr_reg;
2957 int bus, ret;
c3361f2f 2958 bool new_ext, ext;
091d42e4
JR		 2959
2960 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2961 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2962 new_ext = !!ecap_ecs(iommu->ecap);
2963
2964 /*
2965 * The RTT bit can only be changed when translation is disabled,
2966 * but disabling translation means to open a window for data
2967 * corruption. So bail out and don't copy anything if we would
2968 * have to change the bit.
2969 */
2970 if (new_ext != ext)
2971 return -EINVAL;
091d42e4
JR		 2972
2973 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2974 if (!old_rt_phys)
2975 return -EINVAL;
2976
2977 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2978 if (!old_rt)
2979 return -ENOMEM;
2980
2981 /* This is too big for the stack - allocate it from slab */
2982 ctxt_table_entries = ext ? 512 : 256;
2983 ret = -ENOMEM;
2984 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2985 if (!ctxt_tbls)
2986 goto out_unmap;
2987
2988 for (bus = 0; bus < 256; bus++) {
2989 ret = copy_context_table(iommu, &old_rt[bus],
2990 ctxt_tbls, bus, ext);
2991 if (ret) {
2992 pr_err("%s: Failed to copy context table for bus %d\n",
2993 iommu->name, bus);
2994 continue;
2995 }
2996 }
2997
2998 spin_lock_irqsave(&iommu->lock, flags);
2999
3000 /* Context tables are copied, now write them to the root_entry table */
3001 for (bus = 0; bus < 256; bus++) {
3002 int idx = ext ? bus * 2 : bus;
3003 u64 val;
3004
3005 if (ctxt_tbls[idx]) {
3006 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3007 iommu->root_entry[bus].lo = val;
3008 }
3009
3010 if (!ext || !ctxt_tbls[idx + 1])
3011 continue;
3012
3013 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3014 iommu->root_entry[bus].hi = val;
3015 }
3016
3017 spin_unlock_irqrestore(&iommu->lock, flags);
3018
3019 kfree(ctxt_tbls);
3020
3021 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3022
3023 ret = 0;
3024
3025out_unmap:
3026 iounmap(old_rt);
3027
3028 return ret;
3029}
3030
b779260b 3031static int __init init_dmars(void)
ba395927
KA		 3032{
3033 struct dmar_drhd_unit *drhd;
3034 struct dmar_rmrr_unit *rmrr;
a87f4918 3035 bool copied_tables = false;
832bd858 3036 struct device *dev;
ba395927 3037 struct intel_iommu *iommu;
9d783ba0 3038 int i, ret;
2c2e2c38 3039
ba395927
KA		 3040 /*
3041 * for each drhd
3042 * allocate root
3043 * initialize and program root entry to not present
3044 * endfor
3045 */
3046 for_each_drhd_unit(drhd) {
5e0d2a6f 3047 /*
3048 * lock not needed as this is only incremented in the single
3049 * threaded kernel __init code path all other access are read
3050 * only
3051 */
78d8e704 3052 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3053 g_num_of_iommus++;
3054 continue;
3055 }
9f10e5bf 3056 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3057 }
3058
ffebeb46
JL		 3059 /* Preallocate enough resources for IOMMU hot-addition */
3060 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3061 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3062
d9630fe9
WH		 3063 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3064 GFP_KERNEL);
3065 if (!g_iommus) {
9f10e5bf 3066 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3067 ret = -ENOMEM;
3068 goto error;
3069 }
3070
80b20dd8 3071 deferred_flush = kzalloc(g_num_of_iommus *
3072 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3073 if (!deferred_flush) {
5e0d2a6f 3074 ret = -ENOMEM;
989d51fc 3075 goto free_g_iommus;
5e0d2a6f 3076 }
3077
7c919779 3078 for_each_active_iommu(iommu, drhd) {
d9630fe9 3079 g_iommus[iommu->seq_id] = iommu;
ba395927 3080
b63d80d1
JR		 3081 intel_iommu_init_qi(iommu);
3082
e61d98d8
SS		 3083 ret = iommu_init_domains(iommu);
3084 if (ret)
989d51fc 3085 goto free_iommu;
e61d98d8 3086
4158c2ec
JR		 3087 init_translation_status(iommu);
3088
091d42e4
JR		 3089 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3090 iommu_disable_translation(iommu);
3091 clear_translation_pre_enabled(iommu);
3092 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3093 iommu->name);
3094 }
4158c2ec 3095
ba395927
KA		 3096 /*
3097 * TBD:
3098 * we could share the same root & context tables
25985edc 3099 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3100 */
3101 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3102 if (ret)
989d51fc 3103 goto free_iommu;
5f0a7f76 3104
091d42e4
JR		 3105 if (translation_pre_enabled(iommu)) {
3106 pr_info("Translation already enabled - trying to copy translation structures\n");
3107
3108 ret = copy_translation_tables(iommu);
3109 if (ret) {
3110 /*
3111 * We found the IOMMU with translation
3112 * enabled - but failed to copy over the
3113 * old root-entry table. Try to proceed
3114 * by disabling translation now and
3115 * allocating a clean root-entry table.
3116 * This might cause DMAR faults, but
3117 * probably the dump will still succeed.
3118 */
3119 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3120 iommu->name);
3121 iommu_disable_translation(iommu);
3122 clear_translation_pre_enabled(iommu);
3123 } else {
3124 pr_info("Copied translation tables from previous kernel for %s\n",
3125 iommu->name);
a87f4918 3126 copied_tables = true;
091d42e4
JR		 3127 }
3128 }
3129
5f0a7f76
JR		 3130 iommu_flush_write_buffer(iommu);
3131 iommu_set_root_entry(iommu);
3132 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3133 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3134
4ed0d3e6 3135 if (!ecap_pass_through(iommu->ecap))
19943b0e 3136 hw_pass_through = 0;
ba395927
KA		 3137 }
3138
19943b0e 3139 if (iommu_pass_through)
e0fc7e0b
DW		 3140 iommu_identity_mapping |= IDENTMAP_ALL;
3141
d3f13810 3142#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3143 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3144#endif
e0fc7e0b 3145
86080ccc
JR		 3146 if (iommu_identity_mapping) {
3147 ret = si_domain_init(hw_pass_through);
3148 if (ret)
3149 goto free_iommu;
3150 }
3151
e0fc7e0b
DW		 3152 check_tylersburg_isoch();
3153
a87f4918
JR		 3154 /*
3155 * If we copied translations from a previous kernel in the kdump
3156 * case, we can not assign the devices to domains now, as that
3157 * would eliminate the old mappings. So skip this part and defer
3158 * the assignment to device driver initialization time.
3159 */
3160 if (copied_tables)
3161 goto domains_done;
3162
ba395927 3163 /*
19943b0e
DW		 3164 * If pass through is not set or not enabled, setup context entries for
3165 * identity mappings for rmrr, gfx, and isa and may fall back to static
3166 * identity mapping if iommu_identity_mapping is set.
ba395927 3167 */
19943b0e
DW		 3168 if (iommu_identity_mapping) {
3169 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3170 if (ret) {
9f10e5bf 3171 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3172 goto free_iommu;
ba395927
KA		 3173 }
3174 }
ba395927 3175 /*
19943b0e
DW		 3176 * For each rmrr
3177 * for each dev attached to rmrr
3178 * do
3179 * locate drhd for dev, alloc domain for dev
3180 * allocate free domain
3181 * allocate page table entries for rmrr
3182 * if context not allocated for bus
3183 * allocate and init context
3184 * set present in root table for this bus
3185 * init context with domain, translation etc
3186 * endfor
3187 * endfor
ba395927 3188 */
9f10e5bf 3189 pr_info("Setting RMRR:\n");
19943b0e 3190 for_each_rmrr_units(rmrr) {
b683b230
JL		 3191 /* some BIOS lists non-exist devices in DMAR table. */
3192 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3193 i, dev) {
0b9d9753 3194 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3195 if (ret)
9f10e5bf 3196 pr_err("Mapping reserved region failed\n");
ba395927 3197 }
4ed0d3e6 3198 }
49a0429e 3199
19943b0e
DW		 3200 iommu_prepare_isa();
3201
a87f4918
JR		 3202domains_done:
3203
ba395927
KA		 3204 /*
3205 * for each drhd
3206 * enable fault log
3207 * global invalidate context cache
3208 * global invalidate iotlb
3209 * enable translation
3210 */
7c919779 3211 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3212 if (drhd->ignored) {
3213 /*
3214 * we always have to disable PMRs or DMA may fail on
3215 * this device
3216 */
3217 if (force_on)
7c919779 3218 iommu_disable_protect_mem_regions(iommu);
ba395927 3219 continue;
51a63e67 3220 }
ba395927
KA		 3221
3222 iommu_flush_write_buffer(iommu);
3223
3460a6d9
KA		 3224 ret = dmar_set_interrupt(iommu);
3225 if (ret)
989d51fc 3226 goto free_iommu;
3460a6d9 3227
8939ddf6
JR		 3228 if (!translation_pre_enabled(iommu))
3229 iommu_enable_translation(iommu);
3230
b94996c9 3231 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3232 }
3233
3234 return 0;
989d51fc
JL		 3235
3236free_iommu:
ffebeb46
JL		 3237 for_each_active_iommu(iommu, drhd) {
3238 disable_dmar_iommu(iommu);
a868e6b7 3239 free_dmar_iommu(iommu);
ffebeb46 3240 }
9bdc531e 3241 kfree(deferred_flush);
989d51fc 3242free_g_iommus:
d9630fe9 3243 kfree(g_iommus);
989d51fc 3244error:
ba395927
KA		 3245 return ret;
3246}
3247
5a5e02a6 3248/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3249static struct iova *intel_alloc_iova(struct device *dev,
3250 struct dmar_domain *domain,
3251 unsigned long nrpages, uint64_t dma_mask)
ba395927 3252{
ba395927 3253 struct iova *iova = NULL;
ba395927 3254
875764de
DW		 3255 /* Restrict dma_mask to the width that the iommu can handle */
3256 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3257
3258 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3259 /*
3260 * First try to allocate an io virtual address in
284901a9 3261 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3262 * from higher range
ba395927 3263 */
875764de
DW		 3264 iova = alloc_iova(&domain->iovad, nrpages,
3265 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3266 if (iova)
3267 return iova;
3268 }
3269 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3270 if (unlikely(!iova)) {
9f10e5bf 3271 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3272 nrpages, dev_name(dev));
f76aec76
KA		 3273 return NULL;
3274 }
3275
3276 return iova;
3277}
3278
d4b709f4 3279static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3280{
3281 struct dmar_domain *domain;
3282 int ret;
3283
d4b709f4 3284 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3285 if (!domain) {
9f10e5bf 3286 pr_err("Allocating domain for %s failed\n",
d4b709f4 3287 dev_name(dev));
4fe05bbc 3288 return NULL;
ba395927
KA		 3289 }
3290
3291 /* make sure context mapping is ok */
d4b709f4 3292 if (unlikely(!domain_context_mapped(dev))) {
28ccce0d 3293 ret = domain_context_mapping(domain, dev);
f76aec76 3294 if (ret) {
9f10e5bf 3295 pr_err("Domain context map for %s failed\n",
d4b709f4 3296 dev_name(dev));
4fe05bbc 3297 return NULL;
f76aec76 3298 }
ba395927
KA		 3299 }
3300
f76aec76
KA		 3301 return domain;
3302}
3303
d4b709f4 3304static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3305{
3306 struct device_domain_info *info;
3307
3308 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3309 info = dev->archdata.iommu;
147202aa
DW		 3310 if (likely(info))
3311 return info->domain;
3312
3313 return __get_valid_domain_for_dev(dev);
3314}
3315
ecb509ec 3316/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3317static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3318{
3319 int found;
3320
3d89194a 3321 if (iommu_dummy(dev))
1e4c64c4
DW		 3322 return 1;
3323
2c2e2c38 3324 if (!iommu_identity_mapping)
1e4c64c4 3325 return 0;
2c2e2c38 3326
9b226624 3327 found = identity_mapping(dev);
2c2e2c38 3328 if (found) {
ecb509ec 3329 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3330 return 1;
3331 else {
3332 /*
3333 * 32 bit DMA is removed from si_domain and fall back
3334 * to non-identity mapping.
3335 */
bf9c9eda 3336 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3337 pr_info("32bit %s uses non-identity mapping\n",
3338 dev_name(dev));
2c2e2c38
FY		 3339 return 0;
3340 }
3341 } else {
3342 /*
3343 * In case of a detached 64 bit DMA device from vm, the device
3344 * is put into si_domain for identity mapping.
3345 */
ecb509ec 3346 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3347 int ret;
28ccce0d 3348 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3349 if (!ret) {
9f10e5bf
JR		 3350 pr_info("64bit %s uses identity mapping\n",
3351 dev_name(dev));
2c2e2c38
FY		 3352 return 1;
3353 }
3354 }
3355 }
3356
1e4c64c4 3357 return 0;
2c2e2c38
FY		 3358}
3359
5040a918 3360static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3361 size_t size, int dir, u64 dma_mask)
f76aec76 3362{
f76aec76 3363 struct dmar_domain *domain;
5b6985ce 3364 phys_addr_t start_paddr;
f76aec76
KA		 3365 struct iova *iova;
3366 int prot = 0;
6865f0d1 3367 int ret;
8c11e798 3368 struct intel_iommu *iommu;
33041ec0 3369 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3370
3371 BUG_ON(dir == DMA_NONE);
2c2e2c38 3372
5040a918 3373 if (iommu_no_mapping(dev))
6865f0d1 3374 return paddr;
f76aec76 3375
5040a918 3376 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3377 if (!domain)
3378 return 0;
3379
8c11e798 3380 iommu = domain_get_iommu(domain);
88cb6a74 3381 size = aligned_nrpages(paddr, size);
f76aec76 3382
5040a918 3383 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3384 if (!iova)
3385 goto error;
3386
ba395927
KA		 3387 /*
3388 * Check if DMAR supports zero-length reads on write only
3389 * mappings..
3390 */
3391 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3392 !cap_zlr(iommu->cap))
ba395927
KA		 3393 prot |= DMA_PTE_READ;
3394 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3395 prot |= DMA_PTE_WRITE;
3396 /*
6865f0d1 3397 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3398 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3399 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3400 * is not a big problem
3401 */
0ab36de2 3402 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3403 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3404 if (ret)
3405 goto error;
3406
1f0ef2aa
DW		 3407 /* it's a non-present to present mapping. Only flush if caching mode */
3408 if (cap_caching_mode(iommu->cap))
a1ddcbe9
JR		 3409 iommu_flush_iotlb_psi(iommu, domain,
3410 mm_to_dma_pfn(iova->pfn_lo),
3411 size, 0, 1);
1f0ef2aa 3412 else
8c11e798 3413 iommu_flush_write_buffer(iommu);
f76aec76 3414
03d6a246
DW		 3415 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3416 start_paddr += paddr & ~PAGE_MASK;
3417 return start_paddr;
ba395927 3418
ba395927 3419error:
f76aec76
KA		 3420 if (iova)
3421 __free_iova(&domain->iovad, iova);
9f10e5bf 3422 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3423 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3424 return 0;
3425}
3426
ffbbef5c
FT		 3427static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3428 unsigned long offset, size_t size,
3429 enum dma_data_direction dir,
3430 struct dma_attrs *attrs)
bb9e6d65 3431{
ffbbef5c 3432 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3433 dir, *dev->dma_mask);
bb9e6d65
FT		 3434}
3435
5e0d2a6f 3436static void flush_unmaps(void)
3437{
80b20dd8 3438 int i, j;
5e0d2a6f 3439
5e0d2a6f 3440 timer_on = 0;
3441
3442 /* just flush them all */
3443 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3444 struct intel_iommu *iommu = g_iommus[i];
3445 if (!iommu)
3446 continue;
c42d9f32 3447
9dd2fe89
YZ		 3448 if (!deferred_flush[i].next)
3449 continue;
3450
78d5f0f5
NA		 3451 /* In caching mode, global flushes turn emulation expensive */
3452 if (!cap_caching_mode(iommu->cap))
3453 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3454 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3455 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3456 unsigned long mask;
3457 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3458 struct dmar_domain *domain = deferred_flush[i].domain[j];
3459
3460 /* On real hardware multiple invalidations are expensive */
3461 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3462 iommu_flush_iotlb_psi(iommu, domain,
a156ef99 3463 iova->pfn_lo, iova_size(iova),
ea8ea460 3464 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3465 else {
a156ef99 3466 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3467 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3468 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3469 }
93a23a72 3470 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3471 if (deferred_flush[i].freelist[j])
3472 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3473 }
9dd2fe89 3474 deferred_flush[i].next = 0;
5e0d2a6f 3475 }
3476
5e0d2a6f 3477 list_size = 0;
5e0d2a6f 3478}
3479
3480static void flush_unmaps_timeout(unsigned long data)
3481{
80b20dd8 3482 unsigned long flags;
3483
3484 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3485 flush_unmaps();
80b20dd8 3486 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3487}
3488
ea8ea460 3489static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3490{
3491 unsigned long flags;
80b20dd8 3492 int next, iommu_id;
8c11e798 3493 struct intel_iommu *iommu;
5e0d2a6f 3494
3495 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3496 if (list_size == HIGH_WATER_MARK)
3497 flush_unmaps();
3498
8c11e798
WH		 3499 iommu = domain_get_iommu(dom);
3500 iommu_id = iommu->seq_id;
c42d9f32 3501
80b20dd8 3502 next = deferred_flush[iommu_id].next;
3503 deferred_flush[iommu_id].domain[next] = dom;
3504 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3505 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3506 deferred_flush[iommu_id].next++;
5e0d2a6f 3507
3508 if (!timer_on) {
3509 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3510 timer_on = 1;
3511 }
3512 list_size++;
3513 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3514}
3515
d41a4adb 3516static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3517{
f76aec76 3518 struct dmar_domain *domain;
d794dc9b 3519 unsigned long start_pfn, last_pfn;
ba395927 3520 struct iova *iova;
8c11e798 3521 struct intel_iommu *iommu;
ea8ea460 3522 struct page *freelist;
ba395927 3523
73676832 3524 if (iommu_no_mapping(dev))
f76aec76 3525 return;
2c2e2c38 3526
1525a29a 3527 domain = find_domain(dev);
ba395927
KA		 3528 BUG_ON(!domain);
3529
8c11e798
WH		 3530 iommu = domain_get_iommu(domain);
3531
ba395927 3532 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3533 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3534 (unsigned long long)dev_addr))
ba395927 3535 return;
ba395927 3536
d794dc9b
DW		 3537 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3538 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3539
d794dc9b 3540 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3541 dev_name(dev), start_pfn, last_pfn);
ba395927 3542
ea8ea460 3543 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3544
5e0d2a6f 3545 if (intel_iommu_strict) {
a1ddcbe9 3546 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
ea8ea460 3547 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3548 /* free iova */
3549 __free_iova(&domain->iovad, iova);
ea8ea460 3550 dma_free_pagelist(freelist);
5e0d2a6f 3551 } else {
ea8ea460 3552 add_unmap(domain, iova, freelist);
5e0d2a6f 3553 /*
3554 * queue up the release of the unmap to save the 1/6th of the
3555 * cpu used up by the iotlb flush operation...
3556 */
5e0d2a6f 3557 }
ba395927
KA		 3558}
3559
d41a4adb
JL		 3560static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3561 size_t size, enum dma_data_direction dir,
3562 struct dma_attrs *attrs)
3563{
3564 intel_unmap(dev, dev_addr);
3565}
3566
5040a918 3567static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3568 dma_addr_t *dma_handle, gfp_t flags,
3569 struct dma_attrs *attrs)
ba395927 3570{
36746436 3571 struct page *page = NULL;
ba395927
KA		 3572 int order;
3573
5b6985ce 3574 size = PAGE_ALIGN(size);
ba395927 3575 order = get_order(size);
e8bb910d 3576
5040a918 3577 if (!iommu_no_mapping(dev))
e8bb910d 3578 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3579 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3580 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3581 flags |= GFP_DMA;
3582 else
3583 flags |= GFP_DMA32;
3584 }
ba395927 3585
36746436
AM		 3586 if (flags & __GFP_WAIT) {
3587 unsigned int count = size >> PAGE_SHIFT;
3588
3589 page = dma_alloc_from_contiguous(dev, count, order);
3590 if (page && iommu_no_mapping(dev) &&
3591 page_to_phys(page) + size > dev->coherent_dma_mask) {
3592 dma_release_from_contiguous(dev, page, count);
3593 page = NULL;
3594 }
3595 }
3596
3597 if (!page)
3598 page = alloc_pages(flags, order);
3599 if (!page)
ba395927 3600 return NULL;
36746436 3601 memset(page_address(page), 0, size);
ba395927 3602
36746436 3603 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3604 DMA_BIDIRECTIONAL,
5040a918 3605 dev->coherent_dma_mask);
ba395927 3606 if (*dma_handle)
36746436
AM		 3607 return page_address(page);
3608 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3609 __free_pages(page, order);
3610
ba395927
KA		 3611 return NULL;
3612}
3613
5040a918 3614static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3615 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3616{
3617 int order;
36746436 3618 struct page *page = virt_to_page(vaddr);
ba395927 3619
5b6985ce 3620 size = PAGE_ALIGN(size);
ba395927
KA		 3621 order = get_order(size);
3622
d41a4adb 3623 intel_unmap(dev, dma_handle);
36746436
AM		 3624 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3625 __free_pages(page, order);
ba395927
KA		 3626}
3627
5040a918 3628static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3629 int nelems, enum dma_data_direction dir,
3630 struct dma_attrs *attrs)
ba395927 3631{
d41a4adb 3632 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3633}
3634
ba395927 3635static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3636 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3637{
3638 int i;
c03ab37c 3639 struct scatterlist *sg;
ba395927 3640
c03ab37c 3641 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3642 BUG_ON(!sg_page(sg));
4cf2e75d 3643 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3644 sg->dma_length = sg->length;
ba395927
KA		 3645 }
3646 return nelems;
3647}
3648
5040a918 3649static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3650 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3651{
ba395927 3652 int i;
ba395927 3653 struct dmar_domain *domain;
f76aec76
KA		 3654 size_t size = 0;
3655 int prot = 0;
f76aec76
KA		 3656 struct iova *iova = NULL;
3657 int ret;
c03ab37c 3658 struct scatterlist *sg;
b536d24d 3659 unsigned long start_vpfn;
8c11e798 3660 struct intel_iommu *iommu;
ba395927
KA		 3661
3662 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3663 if (iommu_no_mapping(dev))
3664 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3665
5040a918 3666 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3667 if (!domain)
3668 return 0;
3669
8c11e798
WH		 3670 iommu = domain_get_iommu(domain);
3671
b536d24d 3672 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3673 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3674
5040a918
DW		 3675 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3676 *dev->dma_mask);
f76aec76 3677 if (!iova) {
c03ab37c 3678 sglist->dma_length = 0;
f76aec76
KA		 3679 return 0;
3680 }
3681
3682 /*
3683 * Check if DMAR supports zero-length reads on write only
3684 * mappings..
3685 */
3686 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3687 !cap_zlr(iommu->cap))
f76aec76
KA		 3688 prot |= DMA_PTE_READ;
3689 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3690 prot |= DMA_PTE_WRITE;
3691
b536d24d 3692 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3693
f532959b 3694 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3695 if (unlikely(ret)) {
e1605495
DW		 3696 dma_pte_free_pagetable(domain, start_vpfn,
3697 start_vpfn + size - 1);
e1605495
DW		 3698 __free_iova(&domain->iovad, iova);
3699 return 0;
ba395927
KA		 3700 }
3701
1f0ef2aa
DW		 3702 /* it's a non-present to present mapping. Only flush if caching mode */
3703 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3704 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3705 else
8c11e798 3706 iommu_flush_write_buffer(iommu);
1f0ef2aa 3707
ba395927
KA		 3708 return nelems;
3709}
3710
dfb805e8
FT		 3711static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3712{
3713 return !dma_addr;
3714}
3715
160c1d8e 3716struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3717 .alloc = intel_alloc_coherent,
3718 .free = intel_free_coherent,
ba395927
KA		 3719 .map_sg = intel_map_sg,
3720 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3721 .map_page = intel_map_page,
3722 .unmap_page = intel_unmap_page,
dfb805e8 3723 .mapping_error = intel_mapping_error,
ba395927
KA		 3724};
3725
3726static inline int iommu_domain_cache_init(void)
3727{
3728 int ret = 0;
3729
3730 iommu_domain_cache = kmem_cache_create("iommu_domain",
3731 sizeof(struct dmar_domain),
3732 0,
3733 SLAB_HWCACHE_ALIGN,
3734
3735 NULL);
3736 if (!iommu_domain_cache) {
9f10e5bf 3737 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3738 ret = -ENOMEM;
3739 }
3740
3741 return ret;
3742}
3743
3744static inline int iommu_devinfo_cache_init(void)
3745{
3746 int ret = 0;
3747
3748 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3749 sizeof(struct device_domain_info),
3750 0,
3751 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3752 NULL);
3753 if (!iommu_devinfo_cache) {
9f10e5bf 3754 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3755 ret = -ENOMEM;
3756 }
3757
3758 return ret;
3759}
3760
ba395927
KA		 3761static int __init iommu_init_mempool(void)
3762{
3763 int ret;
3764 ret = iommu_iova_cache_init();
3765 if (ret)
3766 return ret;
3767
3768 ret = iommu_domain_cache_init();
3769 if (ret)
3770 goto domain_error;
3771
3772 ret = iommu_devinfo_cache_init();
3773 if (!ret)
3774 return ret;
3775
3776 kmem_cache_destroy(iommu_domain_cache);
3777domain_error:
85b45456 3778 iommu_iova_cache_destroy();
ba395927
KA		 3779
3780 return -ENOMEM;
3781}
3782
3783static void __init iommu_exit_mempool(void)
3784{
3785 kmem_cache_destroy(iommu_devinfo_cache);
3786 kmem_cache_destroy(iommu_domain_cache);
85b45456 3787 iommu_iova_cache_destroy();
ba395927
KA		 3788}
3789
556ab45f
DW		 3790static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3791{
3792 struct dmar_drhd_unit *drhd;
3793 u32 vtbar;
3794 int rc;
3795
3796 /* We know that this device on this chipset has its own IOMMU.
3797 * If we find it under a different IOMMU, then the BIOS is lying
3798 * to us. Hope that the IOMMU for this device is actually
3799 * disabled, and it needs no translation...
3800 */
3801 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3802 if (rc) {
3803 /* "can't" happen */
3804 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3805 return;
3806 }
3807 vtbar &= 0xffff0000;
3808
3809 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3810 drhd = dmar_find_matched_drhd_unit(pdev);
3811 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3812 TAINT_FIRMWARE_WORKAROUND,
3813 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3814 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3815}
3816DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3817
ba395927
KA		 3818static void __init init_no_remapping_devices(void)
3819{
3820 struct dmar_drhd_unit *drhd;
832bd858 3821 struct device *dev;
b683b230 3822 int i;
ba395927
KA		 3823
3824 for_each_drhd_unit(drhd) {
3825 if (!drhd->include_all) {
b683b230
JL		 3826 for_each_active_dev_scope(drhd->devices,
3827 drhd->devices_cnt, i, dev)
3828 break;
832bd858 3829 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3830 if (i == drhd->devices_cnt)
3831 drhd->ignored = 1;
3832 }
3833 }
3834
7c919779 3835 for_each_active_drhd_unit(drhd) {
7c919779 3836 if (drhd->include_all)
ba395927
KA		 3837 continue;
3838
b683b230
JL		 3839 for_each_active_dev_scope(drhd->devices,
3840 drhd->devices_cnt, i, dev)
832bd858 3841 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3842 break;
ba395927
KA		 3843 if (i < drhd->devices_cnt)
3844 continue;
3845
c0771df8
DW		 3846 /* This IOMMU has *only* gfx devices. Either bypass it or
3847 set the gfx_mapped flag, as appropriate */
3848 if (dmar_map_gfx) {
3849 intel_iommu_gfx_mapped = 1;
3850 } else {
3851 drhd->ignored = 1;
b683b230
JL		 3852 for_each_active_dev_scope(drhd->devices,
3853 drhd->devices_cnt, i, dev)
832bd858 3854 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3855 }
3856 }
3857}
3858
f59c7b69
FY		 3859#ifdef CONFIG_SUSPEND
3860static int init_iommu_hw(void)
3861{
3862 struct dmar_drhd_unit *drhd;
3863 struct intel_iommu *iommu = NULL;
3864
3865 for_each_active_iommu(iommu, drhd)
3866 if (iommu->qi)
3867 dmar_reenable_qi(iommu);
3868
b779260b
JC		 3869 for_each_iommu(iommu, drhd) {
3870 if (drhd->ignored) {
3871 /*
3872 * we always have to disable PMRs or DMA may fail on
3873 * this device
3874 */
3875 if (force_on)
3876 iommu_disable_protect_mem_regions(iommu);
3877 continue;
3878 }
3879
f59c7b69
FY		 3880 iommu_flush_write_buffer(iommu);
3881
3882 iommu_set_root_entry(iommu);
3883
3884 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3885 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3886 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3887 iommu_enable_translation(iommu);
b94996c9 3888 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3889 }
3890
3891 return 0;
3892}
3893
3894static void iommu_flush_all(void)
3895{
3896 struct dmar_drhd_unit *drhd;
3897 struct intel_iommu *iommu;
3898
3899 for_each_active_iommu(iommu, drhd) {
3900 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3901 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3902 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3903 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3904 }
3905}
3906
134fac3f 3907static int iommu_suspend(void)
f59c7b69
FY		 3908{
3909 struct dmar_drhd_unit *drhd;
3910 struct intel_iommu *iommu = NULL;
3911 unsigned long flag;
3912
3913 for_each_active_iommu(iommu, drhd) {
3914 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3915 GFP_ATOMIC);
3916 if (!iommu->iommu_state)
3917 goto nomem;
3918 }
3919
3920 iommu_flush_all();
3921
3922 for_each_active_iommu(iommu, drhd) {
3923 iommu_disable_translation(iommu);
3924
1f5b3c3f 3925 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3926
3927 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3928 readl(iommu->reg + DMAR_FECTL_REG);
3929 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3930 readl(iommu->reg + DMAR_FEDATA_REG);
3931 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3932 readl(iommu->reg + DMAR_FEADDR_REG);
3933 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3934 readl(iommu->reg + DMAR_FEUADDR_REG);
3935
1f5b3c3f 3936 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3937 }
3938 return 0;
3939
3940nomem:
3941 for_each_active_iommu(iommu, drhd)
3942 kfree(iommu->iommu_state);
3943
3944 return -ENOMEM;
3945}
3946
134fac3f 3947static void iommu_resume(void)
f59c7b69
FY		 3948{
3949 struct dmar_drhd_unit *drhd;
3950 struct intel_iommu *iommu = NULL;
3951 unsigned long flag;
3952
3953 if (init_iommu_hw()) {
b779260b
JC		 3954 if (force_on)
3955 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3956 else
3957 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3958 return;
f59c7b69
FY		 3959 }
3960
3961 for_each_active_iommu(iommu, drhd) {
3962
1f5b3c3f 3963 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3964
3965 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3966 iommu->reg + DMAR_FECTL_REG);
3967 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3968 iommu->reg + DMAR_FEDATA_REG);
3969 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3970 iommu->reg + DMAR_FEADDR_REG);
3971 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3972 iommu->reg + DMAR_FEUADDR_REG);
3973
1f5b3c3f 3974 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3975 }
3976
3977 for_each_active_iommu(iommu, drhd)
3978 kfree(iommu->iommu_state);
f59c7b69
FY		 3979}
3980
134fac3f 3981static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3982 .resume = iommu_resume,
3983 .suspend = iommu_suspend,
3984};
3985
134fac3f 3986static void __init init_iommu_pm_ops(void)
f59c7b69 3987{
134fac3f 3988 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3989}
3990
3991#else
99592ba4 3992static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3993#endif /* CONFIG_PM */
3994
318fe7df 3995
c2a0b538 3996int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3997{
3998 struct acpi_dmar_reserved_memory *rmrr;
3999 struct dmar_rmrr_unit *rmrru;
4000
4001 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4002 if (!rmrru)
4003 return -ENOMEM;
4004
4005 rmrru->hdr = header;
4006 rmrr = (struct acpi_dmar_reserved_memory *)header;
4007 rmrru->base_address = rmrr->base_address;
4008 rmrru->end_address = rmrr->end_address;
2e455289
JL		 4009 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4010 ((void *)rmrr) + rmrr->header.length,
4011 &rmrru->devices_cnt);
4012 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4013 kfree(rmrru);
4014 return -ENOMEM;
4015 }
318fe7df 4016
2e455289 4017 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4018
2e455289 4019 return 0;
318fe7df
SS		 4020}
4021
6b197249
JL		 4022static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4023{
4024 struct dmar_atsr_unit *atsru;
4025 struct acpi_dmar_atsr *tmp;
4026
4027 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4028 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4029 if (atsr->segment != tmp->segment)
4030 continue;
4031 if (atsr->header.length != tmp->header.length)
4032 continue;
4033 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4034 return atsru;
4035 }
4036
4037 return NULL;
4038}
4039
4040int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4041{
4042 struct acpi_dmar_atsr *atsr;
4043 struct dmar_atsr_unit *atsru;
4044
6b197249
JL		 4045 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4046 return 0;
4047
318fe7df 4048 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4049 atsru = dmar_find_atsr(atsr);
4050 if (atsru)
4051 return 0;
4052
4053 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4054 if (!atsru)
4055 return -ENOMEM;
4056
6b197249
JL		 4057 /*
4058 * If memory is allocated from slab by ACPI _DSM method, we need to
4059 * copy the memory content because the memory buffer will be freed
4060 * on return.
4061 */
4062 atsru->hdr = (void *)(atsru + 1);
4063 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4064 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4065 if (!atsru->include_all) {
4066 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4067 (void *)atsr + atsr->header.length,
4068 &atsru->devices_cnt);
4069 if (atsru->devices_cnt && atsru->devices == NULL) {
4070 kfree(atsru);
4071 return -ENOMEM;
4072 }
4073 }
318fe7df 4074
0e242612 4075 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4076
4077 return 0;
4078}
4079
9bdc531e
JL		 4080static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4081{
4082 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4083 kfree(atsru);
4084}
4085
6b197249
JL		 4086int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4087{
4088 struct acpi_dmar_atsr *atsr;
4089 struct dmar_atsr_unit *atsru;
4090
4091 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4092 atsru = dmar_find_atsr(atsr);
4093 if (atsru) {
4094 list_del_rcu(&atsru->list);
4095 synchronize_rcu();
4096 intel_iommu_free_atsr(atsru);
4097 }
4098
4099 return 0;
4100}
4101
4102int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4103{
4104 int i;
4105 struct device *dev;
4106 struct acpi_dmar_atsr *atsr;
4107 struct dmar_atsr_unit *atsru;
4108
4109 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4110 atsru = dmar_find_atsr(atsr);
4111 if (!atsru)
4112 return 0;
4113
4114 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4115 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4116 i, dev)
4117 return -EBUSY;
4118
4119 return 0;
4120}
4121
ffebeb46
JL		 4122static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4123{
4124 int sp, ret = 0;
4125 struct intel_iommu *iommu = dmaru->iommu;
4126
4127 if (g_iommus[iommu->seq_id])
4128 return 0;
4129
4130 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4131 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4132 iommu->name);
4133 return -ENXIO;
4134 }
4135 if (!ecap_sc_support(iommu->ecap) &&
4136 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4137 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4138 iommu->name);
4139 return -ENXIO;
4140 }
4141 sp = domain_update_iommu_superpage(iommu) - 1;
4142 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4143 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4144 iommu->name);
4145 return -ENXIO;
4146 }
4147
4148 /*
4149 * Disable translation if already enabled prior to OS handover.
4150 */
4151 if (iommu->gcmd & DMA_GCMD_TE)
4152 iommu_disable_translation(iommu);
4153
4154 g_iommus[iommu->seq_id] = iommu;
4155 ret = iommu_init_domains(iommu);
4156 if (ret == 0)
4157 ret = iommu_alloc_root_entry(iommu);
4158 if (ret)
4159 goto out;
4160
4161 if (dmaru->ignored) {
4162 /*
4163 * we always have to disable PMRs or DMA may fail on this device
4164 */
4165 if (force_on)
4166 iommu_disable_protect_mem_regions(iommu);
4167 return 0;
4168 }
4169
4170 intel_iommu_init_qi(iommu);
4171 iommu_flush_write_buffer(iommu);
4172 ret = dmar_set_interrupt(iommu);
4173 if (ret)
4174 goto disable_iommu;
4175
4176 iommu_set_root_entry(iommu);
4177 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4178 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4179 iommu_enable_translation(iommu);
4180
ffebeb46
JL		 4181 iommu_disable_protect_mem_regions(iommu);
4182 return 0;
4183
4184disable_iommu:
4185 disable_dmar_iommu(iommu);
4186out:
4187 free_dmar_iommu(iommu);
4188 return ret;
4189}
4190
6b197249
JL		 4191int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4192{
ffebeb46
JL		 4193 int ret = 0;
4194 struct intel_iommu *iommu = dmaru->iommu;
4195
4196 if (!intel_iommu_enabled)
4197 return 0;
4198 if (iommu == NULL)
4199 return -EINVAL;
4200
4201 if (insert) {
4202 ret = intel_iommu_add(dmaru);
4203 } else {
4204 disable_dmar_iommu(iommu);
4205 free_dmar_iommu(iommu);
4206 }
4207
4208 return ret;
6b197249
JL		 4209}
4210
9bdc531e
JL		 4211static void intel_iommu_free_dmars(void)
4212{
4213 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4214 struct dmar_atsr_unit *atsru, *atsr_n;
4215
4216 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4217 list_del(&rmrru->list);
4218 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4219 kfree(rmrru);
318fe7df
SS		 4220 }
4221
9bdc531e
JL		 4222 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4223 list_del(&atsru->list);
4224 intel_iommu_free_atsr(atsru);
4225 }
318fe7df
SS		 4226}
4227
4228int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4229{
b683b230 4230 int i, ret = 1;
318fe7df 4231 struct pci_bus *bus;
832bd858
DW		 4232 struct pci_dev *bridge = NULL;
4233 struct device *tmp;
318fe7df
SS		 4234 struct acpi_dmar_atsr *atsr;
4235 struct dmar_atsr_unit *atsru;
4236
4237 dev = pci_physfn(dev);
318fe7df 4238 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4239 bridge = bus->self;
318fe7df 4240 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4241 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4242 return 0;
b5f82ddf 4243 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4244 break;
318fe7df 4245 }
b5f82ddf
JL		 4246 if (!bridge)
4247 return 0;
318fe7df 4248
0e242612 4249 rcu_read_lock();
b5f82ddf
JL		 4250 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4251 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4252 if (atsr->segment != pci_domain_nr(dev->bus))
4253 continue;
4254
b683b230 4255 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4256 if (tmp == &bridge->dev)
b683b230 4257 goto out;
b5f82ddf
JL		 4258
4259 if (atsru->include_all)
b683b230 4260 goto out;
b5f82ddf 4261 }
b683b230
JL		 4262 ret = 0;
4263out:
0e242612 4264 rcu_read_unlock();
318fe7df 4265
b683b230 4266 return ret;
318fe7df
SS		 4267}
4268
59ce0515
JL		 4269int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4270{
4271 int ret = 0;
4272 struct dmar_rmrr_unit *rmrru;
4273 struct dmar_atsr_unit *atsru;
4274 struct acpi_dmar_atsr *atsr;
4275 struct acpi_dmar_reserved_memory *rmrr;
4276
4277 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4278 return 0;
4279
4280 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4281 rmrr = container_of(rmrru->hdr,
4282 struct acpi_dmar_reserved_memory, header);
4283 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4284 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4285 ((void *)rmrr) + rmrr->header.length,
4286 rmrr->segment, rmrru->devices,
4287 rmrru->devices_cnt);
27e24950 4288 if(ret < 0)
59ce0515
JL		 4289 return ret;
4290 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4291 dmar_remove_dev_scope(info, rmrr->segment,
4292 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4293 }
4294 }
4295
4296 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4297 if (atsru->include_all)
4298 continue;
4299
4300 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4301 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4302 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4303 (void *)atsr + atsr->header.length,
4304 atsr->segment, atsru->devices,
4305 atsru->devices_cnt);
4306 if (ret > 0)
4307 break;
4308 else if(ret < 0)
4309 return ret;
4310 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4311 if (dmar_remove_dev_scope(info, atsr->segment,
4312 atsru->devices, atsru->devices_cnt))
4313 break;
4314 }
4315 }
4316
4317 return 0;
4318}
4319
99dcaded
FY		 4320/*
4321 * Here we only respond to action of unbound device from driver.
4322 *
4323 * Added device is not attached to its DMAR domain here yet. That will happen
4324 * when mapping the device to iova.
4325 */
4326static int device_notifier(struct notifier_block *nb,
4327 unsigned long action, void *data)
4328{
4329 struct device *dev = data;
99dcaded
FY		 4330 struct dmar_domain *domain;
4331
3d89194a 4332 if (iommu_dummy(dev))
44cd613c
DW		 4333 return 0;
4334
1196c2fb 4335 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4336 return 0;
4337
1525a29a 4338 domain = find_domain(dev);
99dcaded
FY		 4339 if (!domain)
4340 return 0;
4341
3a5670e8 4342 down_read(&dmar_global_lock);
bf9c9eda 4343 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4344 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4345 domain_exit(domain);
3a5670e8 4346 up_read(&dmar_global_lock);
a97590e5 4347
99dcaded
FY		 4348 return 0;
4349}
4350
4351static struct notifier_block device_nb = {
4352 .notifier_call = device_notifier,
4353};
4354
75f05569
JL		 4355static int intel_iommu_memory_notifier(struct notifier_block *nb,
4356 unsigned long val, void *v)
4357{
4358 struct memory_notify *mhp = v;
4359 unsigned long long start, end;
4360 unsigned long start_vpfn, last_vpfn;
4361
4362 switch (val) {
4363 case MEM_GOING_ONLINE:
4364 start = mhp->start_pfn << PAGE_SHIFT;
4365 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4366 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4367 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4368 start, end);
4369 return NOTIFY_BAD;
4370 }
4371 break;
4372
4373 case MEM_OFFLINE:
4374 case MEM_CANCEL_ONLINE:
4375 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4376 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4377 while (start_vpfn <= last_vpfn) {
4378 struct iova *iova;
4379 struct dmar_drhd_unit *drhd;
4380 struct intel_iommu *iommu;
ea8ea460 4381 struct page *freelist;
75f05569
JL		 4382
4383 iova = find_iova(&si_domain->iovad, start_vpfn);
4384 if (iova == NULL) {
9f10e5bf 4385 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4386 start_vpfn);
4387 break;
4388 }
4389
4390 iova = split_and_remove_iova(&si_domain->iovad, iova,
4391 start_vpfn, last_vpfn);
4392 if (iova == NULL) {
9f10e5bf 4393 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4394 start_vpfn, last_vpfn);
4395 return NOTIFY_BAD;
4396 }
4397
ea8ea460
DW		 4398 freelist = domain_unmap(si_domain, iova->pfn_lo,
4399 iova->pfn_hi);
4400
75f05569
JL		 4401 rcu_read_lock();
4402 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4403 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4404 iova->pfn_lo, iova_size(iova),
ea8ea460 4405 !freelist, 0);
75f05569 4406 rcu_read_unlock();
ea8ea460 4407 dma_free_pagelist(freelist);
75f05569
JL		 4408
4409 start_vpfn = iova->pfn_hi + 1;
4410 free_iova_mem(iova);
4411 }
4412 break;
4413 }
4414
4415 return NOTIFY_OK;
4416}
4417
4418static struct notifier_block intel_iommu_memory_nb = {
4419 .notifier_call = intel_iommu_memory_notifier,
4420 .priority = 0
4421};
4422
a5459cfe
AW		 4423
4424static ssize_t intel_iommu_show_version(struct device *dev,
4425 struct device_attribute *attr,
4426 char *buf)
4427{
4428 struct intel_iommu *iommu = dev_get_drvdata(dev);
4429 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4430 return sprintf(buf, "%d:%d\n",
4431 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4432}
4433static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4434
4435static ssize_t intel_iommu_show_address(struct device *dev,
4436 struct device_attribute *attr,
4437 char *buf)
4438{
4439 struct intel_iommu *iommu = dev_get_drvdata(dev);
4440 return sprintf(buf, "%llx\n", iommu->reg_phys);
4441}
4442static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4443
4444static ssize_t intel_iommu_show_cap(struct device *dev,
4445 struct device_attribute *attr,
4446 char *buf)
4447{
4448 struct intel_iommu *iommu = dev_get_drvdata(dev);
4449 return sprintf(buf, "%llx\n", iommu->cap);
4450}
4451static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4452
4453static ssize_t intel_iommu_show_ecap(struct device *dev,
4454 struct device_attribute *attr,
4455 char *buf)
4456{
4457 struct intel_iommu *iommu = dev_get_drvdata(dev);
4458 return sprintf(buf, "%llx\n", iommu->ecap);
4459}
4460static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4461
2238c082
AW		 4462static ssize_t intel_iommu_show_ndoms(struct device *dev,
4463 struct device_attribute *attr,
4464 char *buf)
4465{
4466 struct intel_iommu *iommu = dev_get_drvdata(dev);
4467 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4468}
4469static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4470
4471static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4472 struct device_attribute *attr,
4473 char *buf)
4474{
4475 struct intel_iommu *iommu = dev_get_drvdata(dev);
4476 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4477 cap_ndoms(iommu->cap)));
4478}
4479static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4480
a5459cfe
AW		 4481static struct attribute *intel_iommu_attrs[] = {
4482 &dev_attr_version.attr,
4483 &dev_attr_address.attr,
4484 &dev_attr_cap.attr,
4485 &dev_attr_ecap.attr,
2238c082
AW		 4486 &dev_attr_domains_supported.attr,
4487 &dev_attr_domains_used.attr,
a5459cfe
AW		 4488 NULL,
4489};
4490
4491static struct attribute_group intel_iommu_group = {
4492 .name = "intel-iommu",
4493 .attrs = intel_iommu_attrs,
4494};
4495
4496const struct attribute_group *intel_iommu_groups[] = {
4497 &intel_iommu_group,
4498 NULL,
4499};
4500
ba395927
KA		 4501int __init intel_iommu_init(void)
4502{
9bdc531e 4503 int ret = -ENODEV;
3a93c841 4504 struct dmar_drhd_unit *drhd;
7c919779 4505 struct intel_iommu *iommu;
ba395927 4506
a59b50e9
JC		 4507 /* VT-d is required for a TXT/tboot launch, so enforce that */
4508 force_on = tboot_force_iommu();
4509
3a5670e8
JL		 4510 if (iommu_init_mempool()) {
4511 if (force_on)
4512 panic("tboot: Failed to initialize iommu memory\n");
4513 return -ENOMEM;
4514 }
4515
4516 down_write(&dmar_global_lock);
a59b50e9
JC		 4517 if (dmar_table_init()) {
4518 if (force_on)
4519 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4520 goto out_free_dmar;
a59b50e9 4521 }
ba395927 4522
c2c7286a 4523 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4524 if (force_on)
4525 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4526 goto out_free_dmar;
a59b50e9 4527 }
1886e8a9 4528
75f1cdf1 4529 if (no_iommu || dmar_disabled)
9bdc531e 4530 goto out_free_dmar;
2ae21010 4531
318fe7df 4532 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4533 pr_info("No RMRR found\n");
318fe7df
SS		 4534
4535 if (list_empty(&dmar_atsr_units))
9f10e5bf 4536 pr_info("No ATSR found\n");
318fe7df 4537
51a63e67
JC		 4538 if (dmar_init_reserved_ranges()) {
4539 if (force_on)
4540 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4541 goto out_free_reserved_range;
51a63e67 4542 }
ba395927
KA		 4543
4544 init_no_remapping_devices();
4545
b779260b 4546 ret = init_dmars();
ba395927 4547 if (ret) {
a59b50e9
JC		 4548 if (force_on)
4549 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4550 pr_err("Initialization failed\n");
9bdc531e 4551 goto out_free_reserved_range;
ba395927 4552 }
3a5670e8 4553 up_write(&dmar_global_lock);
9f10e5bf 4554 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4555
5e0d2a6f 4556 init_timer(&unmap_timer);
75f1cdf1
FT		 4557#ifdef CONFIG_SWIOTLB
4558 swiotlb = 0;
4559#endif
19943b0e 4560 dma_ops = &intel_dma_ops;
4ed0d3e6 4561
134fac3f 4562 init_iommu_pm_ops();
a8bcbb0d 4563
a5459cfe
AW		 4564 for_each_active_iommu(iommu, drhd)
4565 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4566 intel_iommu_groups,
2439d4aa 4567 "%s", iommu->name);
a5459cfe 4568
4236d97d 4569 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4570 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4571 if (si_domain && !hw_pass_through)
4572 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4573
8bc1f85c
ED		 4574 intel_iommu_enabled = 1;
4575
ba395927 4576 return 0;
9bdc531e
JL		 4577
4578out_free_reserved_range:
4579 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4580out_free_dmar:
4581 intel_iommu_free_dmars();
3a5670e8
JL		 4582 up_write(&dmar_global_lock);
4583 iommu_exit_mempool();
9bdc531e 4584 return ret;
ba395927 4585}
e820482c 4586
579305f7
AW		 4587static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4588{
4589 struct intel_iommu *iommu = opaque;
4590
4591 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4592 return 0;
4593}
4594
4595/*
4596 * NB - intel-iommu lacks any sort of reference counting for the users of
4597 * dependent devices. If multiple endpoints have intersecting dependent
4598 * devices, unbinding the driver from any one of them will possibly leave
4599 * the others unable to operate.
4600 */
3199aa6b 4601static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4602 struct device *dev)
3199aa6b 4603{
0bcb3e28 4604 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4605 return;
4606
579305f7 4607 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4608}
4609
2c2e2c38 4610static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4611 struct device *dev)
c7151a8d 4612{
bca2b916 4613 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4614 struct intel_iommu *iommu;
4615 unsigned long flags;
2f119c78 4616 bool found = false;
156baca8 4617 u8 bus, devfn;
c7151a8d 4618
bf9c9eda 4619 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4620 if (!iommu)
4621 return;
4622
4623 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4624 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4625 if (info->iommu == iommu && info->bus == bus &&
4626 info->devfn == devfn) {
109b9b04 4627 unlink_domain_info(info);
c7151a8d
WH		 4628 spin_unlock_irqrestore(&device_domain_lock, flags);
4629
93a23a72 4630 iommu_disable_dev_iotlb(info);
c7151a8d 4631 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4632 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4633 free_devinfo_mem(info);
4634
4635 spin_lock_irqsave(&device_domain_lock, flags);
4636
4637 if (found)
4638 break;
4639 else
4640 continue;
4641 }
4642
4643 /* if there is no other devices under the same iommu
4644 * owned by this domain, clear this iommu in iommu_bmp
4645 * update iommu count and coherency
4646 */
8bbc4410 4647 if (info->iommu == iommu)
2f119c78 4648 found = true;
c7151a8d
WH		 4649 }
4650
3e7abe25
RD		 4651 spin_unlock_irqrestore(&device_domain_lock, flags);
4652
c7151a8d 4653 if (found == 0) {
fb170fb4
JL		 4654 domain_detach_iommu(domain, iommu);
4655 if (!domain_type_is_vm_or_si(domain))
4656 iommu_detach_domain(domain, iommu);
c7151a8d 4657 }
c7151a8d
WH		 4658}
4659
2c2e2c38 4660static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4661{
4662 int adjust_width;
4663
0fb5fe87
RM		 4664 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4665 DMA_32BIT_PFN);
5e98c4b1
WH		 4666 domain_reserve_special_ranges(domain);
4667
4668 /* calculate AGAW */
4669 domain->gaw = guest_width;
4670 adjust_width = guestwidth_to_adjustwidth(guest_width);
4671 domain->agaw = width_to_agaw(adjust_width);
4672
5e98c4b1 4673 domain->iommu_coherency = 0;
c5b15255 4674 domain->iommu_snooping = 0;
6dd9a7c7 4675 domain->iommu_superpage = 0;
fe40f1e0 4676 domain->max_addr = 0;
5e98c4b1
WH		 4677
4678 /* always allocate the top pgd */
4c923d47 4679 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4680 if (!domain->pgd)
4681 return -ENOMEM;
4682 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4683 return 0;
4684}
4685
00a77deb 4686static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4687{
5d450806 4688 struct dmar_domain *dmar_domain;
00a77deb
JR		 4689 struct iommu_domain *domain;
4690
4691 if (type != IOMMU_DOMAIN_UNMANAGED)
4692 return NULL;
38717946 4693
ab8dfe25 4694 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4695 if (!dmar_domain) {
9f10e5bf 4696 pr_err("Can't allocate dmar_domain\n");
00a77deb 4697 return NULL;
38717946 4698 }
2c2e2c38 4699 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4700 pr_err("Domain initialization failed\n");
92d03cc8 4701 domain_exit(dmar_domain);
00a77deb 4702 return NULL;
38717946 4703 }
8140a95d 4704 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4705
00a77deb 4706 domain = &dmar_domain->domain;
8a0e715b
JR		 4707 domain->geometry.aperture_start = 0;
4708 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4709 domain->geometry.force_aperture = true;
4710
00a77deb 4711 return domain;
38717946 4712}
38717946 4713
00a77deb 4714static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4715{
00a77deb 4716 domain_exit(to_dmar_domain(domain));
38717946 4717}
38717946 4718
4c5478c9
JR		 4719static int intel_iommu_attach_device(struct iommu_domain *domain,
4720 struct device *dev)
38717946 4721{
00a77deb 4722 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4723 struct intel_iommu *iommu;
4724 int addr_width;
156baca8 4725 u8 bus, devfn;
faa3d6f5 4726
c875d2c1
AW		 4727 if (device_is_rmrr_locked(dev)) {
4728 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4729 return -EPERM;
4730 }
4731
7207d8f9
DW		 4732 /* normally dev is not mapped */
4733 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4734 struct dmar_domain *old_domain;
4735
1525a29a 4736 old_domain = find_domain(dev);
faa3d6f5 4737 if (old_domain) {
ab8dfe25 4738 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4739 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4740 else
4741 domain_remove_dev_info(old_domain);
62c22167
JR		 4742
4743 if (!domain_type_is_vm_or_si(old_domain) &&
4744 list_empty(&old_domain->devices))
4745 domain_exit(old_domain);
faa3d6f5
WH		 4746 }
4747 }
4748
156baca8 4749 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4750 if (!iommu)
4751 return -ENODEV;
4752
4753 /* check if this iommu agaw is sufficient for max mapped address */
4754 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4755 if (addr_width > cap_mgaw(iommu->cap))
4756 addr_width = cap_mgaw(iommu->cap);
4757
4758 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4759 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4760 "sufficient for the mapped address (%llx)\n",
a99c47a2 4761 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4762 return -EFAULT;
4763 }
a99c47a2
TL		 4764 dmar_domain->gaw = addr_width;
4765
4766 /*
4767 * Knock out extra levels of page tables if necessary
4768 */
4769 while (iommu->agaw < dmar_domain->agaw) {
4770 struct dma_pte *pte;
4771
4772 pte = dmar_domain->pgd;
4773 if (dma_pte_present(pte)) {
25cbff16
SY		 4774 dmar_domain->pgd = (struct dma_pte *)
4775 phys_to_virt(dma_pte_addr(pte));
7a661013 4776 free_pgtable_page(pte);
a99c47a2
TL		 4777 }
4778 dmar_domain->agaw--;
4779 }
fe40f1e0 4780
28ccce0d 4781 return domain_add_dev_info(dmar_domain, dev);
38717946 4782}
38717946 4783
4c5478c9
JR		 4784static void intel_iommu_detach_device(struct iommu_domain *domain,
4785 struct device *dev)
38717946 4786{
00a77deb 4787 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4788}
c7151a8d 4789
b146a1c9
JR		 4790static int intel_iommu_map(struct iommu_domain *domain,
4791 unsigned long iova, phys_addr_t hpa,
5009065d 4792 size_t size, int iommu_prot)
faa3d6f5 4793{
00a77deb 4794 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4795 u64 max_addr;
dde57a21 4796 int prot = 0;
faa3d6f5 4797 int ret;
fe40f1e0 4798
dde57a21
JR		 4799 if (iommu_prot & IOMMU_READ)
4800 prot |= DMA_PTE_READ;
4801 if (iommu_prot & IOMMU_WRITE)
4802 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4803 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4804 prot |= DMA_PTE_SNP;
dde57a21 4805
163cc52c 4806 max_addr = iova + size;
dde57a21 4807 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4808 u64 end;
4809
4810 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4811 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4812 if (end < max_addr) {
9f10e5bf 4813 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4814 "sufficient for the mapped address (%llx)\n",
8954da1f 4815 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4816 return -EFAULT;
4817 }
dde57a21 4818 dmar_domain->max_addr = max_addr;
fe40f1e0 4819 }
ad051221
DW		 4820 /* Round up size to next multiple of PAGE_SIZE, if it and
4821 the low bits of hpa would take us onto the next page */
88cb6a74 4822 size = aligned_nrpages(hpa, size);
ad051221
DW		 4823 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4824 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4825 return ret;
38717946 4826}
38717946 4827
5009065d 4828static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4829 unsigned long iova, size_t size)
38717946 4830{
00a77deb 4831 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4832 struct page *freelist = NULL;
4833 struct intel_iommu *iommu;
4834 unsigned long start_pfn, last_pfn;
4835 unsigned int npages;
4836 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4837
4838 /* Cope with horrid API which requires us to unmap more than the
4839 size argument if it happens to be a large-page mapping. */
4840 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4841 BUG();
4842
4843 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4844 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4845
ea8ea460
DW		 4846 start_pfn = iova >> VTD_PAGE_SHIFT;
4847 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4848
4849 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4850
4851 npages = last_pfn - start_pfn + 1;
4852
4853 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
a1ddcbe9
JR		 4854 iommu = g_iommus[iommu_id];
4855
4856 /*
4857 * find bit position of dmar_domain
4858 */
4859 ndomains = cap_ndoms(iommu->cap);
4860 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4861 if (get_iommu_domain(iommu, num) == dmar_domain)
4862 iommu_flush_iotlb_psi(iommu, dmar_domain,
4863 start_pfn, npages,
4864 !freelist, 0);
4865 }
ea8ea460
DW		 4866
4867 }
4868
4869 dma_free_pagelist(freelist);
fe40f1e0 4870
163cc52c
DW		 4871 if (dmar_domain->max_addr == iova + size)
4872 dmar_domain->max_addr = iova;
b146a1c9 4873
5cf0a76f 4874 return size;
38717946 4875}
38717946 4876
d14d6577 4877static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4878 dma_addr_t iova)
38717946 4879{
00a77deb 4880 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4881 struct dma_pte *pte;
5cf0a76f 4882 int level = 0;
faa3d6f5 4883 u64 phys = 0;
38717946 4884
5cf0a76f 4885 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4886 if (pte)
faa3d6f5 4887 phys = dma_pte_addr(pte);
38717946 4888
faa3d6f5 4889 return phys;
38717946 4890}
a8bcbb0d 4891
5d587b8d 4892static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4893{
dbb9fd86 4894 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4895 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4896 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4897 return irq_remapping_enabled == 1;
dbb9fd86 4898
5d587b8d 4899 return false;
dbb9fd86
SY		 4900}
4901
abdfdde2
AW		 4902static int intel_iommu_add_device(struct device *dev)
4903{
a5459cfe 4904 struct intel_iommu *iommu;
abdfdde2 4905 struct iommu_group *group;
156baca8 4906 u8 bus, devfn;
70ae6f0d 4907
a5459cfe
AW		 4908 iommu = device_to_iommu(dev, &bus, &devfn);
4909 if (!iommu)
70ae6f0d
AW		 4910 return -ENODEV;
4911
a5459cfe 4912 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4913
e17f9ff4 4914 group = iommu_group_get_for_dev(dev);
783f157b 4915
e17f9ff4
AW		 4916 if (IS_ERR(group))
4917 return PTR_ERR(group);
bcb71abe 4918
abdfdde2 4919 iommu_group_put(group);
e17f9ff4 4920 return 0;
abdfdde2 4921}
70ae6f0d 4922
abdfdde2
AW		 4923static void intel_iommu_remove_device(struct device *dev)
4924{
a5459cfe
AW		 4925 struct intel_iommu *iommu;
4926 u8 bus, devfn;
4927
4928 iommu = device_to_iommu(dev, &bus, &devfn);
4929 if (!iommu)
4930 return;
4931
abdfdde2 4932 iommu_group_remove_device(dev);
a5459cfe
AW		 4933
4934 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4935}
4936
b22f6434 4937static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4938 .capable = intel_iommu_capable,
00a77deb
JR		 4939 .domain_alloc = intel_iommu_domain_alloc,
4940 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4941 .attach_dev = intel_iommu_attach_device,
4942 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4943 .map = intel_iommu_map,
4944 .unmap = intel_iommu_unmap,
315786eb 4945 .map_sg = default_iommu_map_sg,
a8bcbb0d 4946 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4947 .add_device = intel_iommu_add_device,
4948 .remove_device = intel_iommu_remove_device,
6d1c56a9 4949 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4950};
9af88143 4951
9452618e
DV		 4952static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4953{
4954 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4955 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4956 dmar_map_gfx = 0;
4957}
4958
4959DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4960DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4961DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4962DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4963DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4964DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4965DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4966
d34d6517 4967static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4968{
4969 /*
4970 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4971 * but needs it. Same seems to hold for the desktop versions.
9af88143 4972 */
9f10e5bf 4973 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4974 rwbf_quirk = 1;
4975}
4976
4977DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4978DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4979DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4980DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4981DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4982DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4983DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4984
eecfd57f
AJ		 4985#define GGC 0x52
4986#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4987#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4988#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4989#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4990#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4991#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4992#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4993#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4994
d34d6517 4995static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4996{
4997 unsigned short ggc;
4998
eecfd57f 4999 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 5000 return;
5001
eecfd57f 5002 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 5003 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 5004 dmar_map_gfx = 0;
6fbcfb3e
DW		 5005 } else if (dmar_map_gfx) {
5006 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 5007 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 5008 intel_iommu_strict = 1;
5009 }
9eecabcb
DW		 5010}
5011DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5012DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5013DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5014DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5015
e0fc7e0b
DW		 5016/* On Tylersburg chipsets, some BIOSes have been known to enable the
5017 ISOCH DMAR unit for the Azalia sound device, but not give it any
5018 TLB entries, which causes it to deadlock. Check for that. We do
5019 this in a function called from init_dmars(), instead of in a PCI
5020 quirk, because we don't want to print the obnoxious "BIOS broken"
5021 message if VT-d is actually disabled.
5022*/
5023static void __init check_tylersburg_isoch(void)
5024{
5025 struct pci_dev *pdev;
5026 uint32_t vtisochctrl;
5027
5028 /* If there's no Azalia in the system anyway, forget it. */
5029 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5030 if (!pdev)
5031 return;
5032 pci_dev_put(pdev);
5033
5034 /* System Management Registers. Might be hidden, in which case
5035 we can't do the sanity check. But that's OK, because the
5036 known-broken BIOSes _don't_ actually hide it, so far. */
5037 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5038 if (!pdev)
5039 return;
5040
5041 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5042 pci_dev_put(pdev);
5043 return;
5044 }
5045
5046 pci_dev_put(pdev);
5047
5048 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5049 if (vtisochctrl & 1)
5050 return;
5051
5052 /* Drop all bits other than the number of TLB entries */
5053 vtisochctrl &= 0x1c;
5054
5055 /* If we have the recommended number of TLB entries (16), fine. */
5056 if (vtisochctrl == 0x10)
5057 return;
5058
5059 /* Zero TLB entries? You get to ride the short bus to school. */
5060 if (!vtisochctrl) {
5061 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5062 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5063 dmi_get_system_info(DMI_BIOS_VENDOR),
5064 dmi_get_system_info(DMI_BIOS_VERSION),
5065 dmi_get_system_info(DMI_PRODUCT_VERSION));
5066 iommu_identity_mapping |= IDENTMAP_AZALIA;
5067 return;
5068 }
9f10e5bf
JR		 5069
5070 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5071 vtisochctrl);
5072}
Linux kernel - Deliverables
This page took 1.527817 seconds and 5 git commands to generate.