2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
26 * These routines are used by both DMA-remapping and Interrupt-remapping
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36 #include <linux/tboot.h>
37 #include <linux/dmi.h>
39 #define PREFIX "DMAR: "
41 /* No locks are needed as DMA remapping hardware unit
42 * list is constructed at boot time and hotplug of
43 * these units are not supported by the architecture.
45 LIST_HEAD(dmar_drhd_units
);
47 static struct acpi_table_header
* __initdata dmar_tbl
;
48 static acpi_size dmar_tbl_size
;
50 static void __init
dmar_register_drhd_unit(struct dmar_drhd_unit
*drhd
)
53 * add INCLUDE_ALL at the tail, so scan the list will find it at
56 if (drhd
->include_all
)
57 list_add_tail(&drhd
->list
, &dmar_drhd_units
);
59 list_add(&drhd
->list
, &dmar_drhd_units
);
62 static int __init
dmar_parse_one_dev_scope(struct acpi_dmar_device_scope
*scope
,
63 struct pci_dev
**dev
, u16 segment
)
66 struct pci_dev
*pdev
= NULL
;
67 struct acpi_dmar_pci_path
*path
;
70 bus
= pci_find_bus(segment
, scope
->bus
);
71 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
72 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
73 / sizeof(struct acpi_dmar_pci_path
);
79 * Some BIOSes list non-exist devices in DMAR table, just
84 PREFIX
"Device scope bus [%d] not found\n",
88 pdev
= pci_get_slot(bus
, PCI_DEVFN(path
->dev
, path
->fn
));
90 printk(KERN_WARNING PREFIX
91 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
92 segment
, bus
->number
, path
->dev
, path
->fn
);
97 bus
= pdev
->subordinate
;
100 printk(KERN_WARNING PREFIX
101 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment
, scope
->bus
, path
->dev
, path
->fn
);
106 if ((scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_ENDPOINT
&& \
107 pdev
->subordinate
) || (scope
->entry_type
== \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE
&& !pdev
->subordinate
)) {
110 printk(KERN_WARNING PREFIX
111 "Device scope type does not match for %s\n",
119 static int __init
dmar_parse_dev_scope(void *start
, void *end
, int *cnt
,
120 struct pci_dev
***devices
, u16 segment
)
122 struct acpi_dmar_device_scope
*scope
;
128 while (start
< end
) {
130 if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_ENDPOINT
||
131 scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_BRIDGE
)
134 printk(KERN_WARNING PREFIX
135 "Unsupported device scope\n");
136 start
+= scope
->length
;
141 *devices
= kcalloc(*cnt
, sizeof(struct pci_dev
*), GFP_KERNEL
);
147 while (start
< end
) {
149 if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_ENDPOINT
||
150 scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_BRIDGE
) {
151 ret
= dmar_parse_one_dev_scope(scope
,
152 &(*devices
)[index
], segment
);
159 start
+= scope
->length
;
166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
167 * structure which uniquely represent one DMA remapping hardware unit
168 * present in the platform
171 dmar_parse_one_drhd(struct acpi_dmar_header
*header
)
173 struct acpi_dmar_hardware_unit
*drhd
;
174 struct dmar_drhd_unit
*dmaru
;
177 drhd
= (struct acpi_dmar_hardware_unit
*)header
;
178 if (!drhd
->address
) {
179 /* Promote an attitude of violence to a BIOS engineer today */
180 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
181 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
182 dmi_get_system_info(DMI_BIOS_VENDOR
),
183 dmi_get_system_info(DMI_BIOS_VERSION
),
184 dmi_get_system_info(DMI_PRODUCT_VERSION
));
187 dmaru
= kzalloc(sizeof(*dmaru
), GFP_KERNEL
);
192 dmaru
->reg_base_addr
= drhd
->address
;
193 dmaru
->segment
= drhd
->segment
;
194 dmaru
->include_all
= drhd
->flags
& 0x1; /* BIT0: INCLUDE_ALL */
196 ret
= alloc_iommu(dmaru
);
201 dmar_register_drhd_unit(dmaru
);
205 static int __init
dmar_parse_dev(struct dmar_drhd_unit
*dmaru
)
207 struct acpi_dmar_hardware_unit
*drhd
;
210 drhd
= (struct acpi_dmar_hardware_unit
*) dmaru
->hdr
;
212 if (dmaru
->include_all
)
215 ret
= dmar_parse_dev_scope((void *)(drhd
+ 1),
216 ((void *)drhd
) + drhd
->header
.length
,
217 &dmaru
->devices_cnt
, &dmaru
->devices
,
220 list_del(&dmaru
->list
);
227 LIST_HEAD(dmar_rmrr_units
);
229 static void __init
dmar_register_rmrr_unit(struct dmar_rmrr_unit
*rmrr
)
231 list_add(&rmrr
->list
, &dmar_rmrr_units
);
236 dmar_parse_one_rmrr(struct acpi_dmar_header
*header
)
238 struct acpi_dmar_reserved_memory
*rmrr
;
239 struct dmar_rmrr_unit
*rmrru
;
241 rmrru
= kzalloc(sizeof(*rmrru
), GFP_KERNEL
);
246 rmrr
= (struct acpi_dmar_reserved_memory
*)header
;
247 rmrru
->base_address
= rmrr
->base_address
;
248 rmrru
->end_address
= rmrr
->end_address
;
250 dmar_register_rmrr_unit(rmrru
);
255 rmrr_parse_dev(struct dmar_rmrr_unit
*rmrru
)
257 struct acpi_dmar_reserved_memory
*rmrr
;
260 rmrr
= (struct acpi_dmar_reserved_memory
*) rmrru
->hdr
;
261 ret
= dmar_parse_dev_scope((void *)(rmrr
+ 1),
262 ((void *)rmrr
) + rmrr
->header
.length
,
263 &rmrru
->devices_cnt
, &rmrru
->devices
, rmrr
->segment
);
265 if (ret
|| (rmrru
->devices_cnt
== 0)) {
266 list_del(&rmrru
->list
);
272 static LIST_HEAD(dmar_atsr_units
);
274 static int __init
dmar_parse_one_atsr(struct acpi_dmar_header
*hdr
)
276 struct acpi_dmar_atsr
*atsr
;
277 struct dmar_atsr_unit
*atsru
;
279 atsr
= container_of(hdr
, struct acpi_dmar_atsr
, header
);
280 atsru
= kzalloc(sizeof(*atsru
), GFP_KERNEL
);
285 atsru
->include_all
= atsr
->flags
& 0x1;
287 list_add(&atsru
->list
, &dmar_atsr_units
);
292 static int __init
atsr_parse_dev(struct dmar_atsr_unit
*atsru
)
295 struct acpi_dmar_atsr
*atsr
;
297 if (atsru
->include_all
)
300 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
301 rc
= dmar_parse_dev_scope((void *)(atsr
+ 1),
302 (void *)atsr
+ atsr
->header
.length
,
303 &atsru
->devices_cnt
, &atsru
->devices
,
305 if (rc
|| !atsru
->devices_cnt
) {
306 list_del(&atsru
->list
);
313 int dmar_find_matched_atsr_unit(struct pci_dev
*dev
)
317 struct acpi_dmar_atsr
*atsr
;
318 struct dmar_atsr_unit
*atsru
;
320 list_for_each_entry(atsru
, &dmar_atsr_units
, list
) {
321 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
322 if (atsr
->segment
== pci_domain_nr(dev
->bus
))
329 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
) {
330 struct pci_dev
*bridge
= bus
->self
;
332 if (!bridge
|| !bridge
->is_pcie
||
333 bridge
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
)
336 if (bridge
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
) {
337 for (i
= 0; i
< atsru
->devices_cnt
; i
++)
338 if (atsru
->devices
[i
] == bridge
)
344 if (atsru
->include_all
)
351 #ifdef CONFIG_ACPI_NUMA
353 dmar_parse_one_rhsa(struct acpi_dmar_header
*header
)
355 struct acpi_dmar_rhsa
*rhsa
;
356 struct dmar_drhd_unit
*drhd
;
358 rhsa
= (struct acpi_dmar_rhsa
*)header
;
359 for_each_drhd_unit(drhd
) {
360 if (drhd
->reg_base_addr
== rhsa
->base_address
) {
361 int node
= acpi_map_pxm_to_node(rhsa
->proximity_domain
);
363 if (!node_online(node
))
365 drhd
->iommu
->node
= node
;
369 WARN(1, "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
370 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
372 dmi_get_system_info(DMI_BIOS_VENDOR
),
373 dmi_get_system_info(DMI_BIOS_VERSION
),
374 dmi_get_system_info(DMI_PRODUCT_VERSION
));
381 dmar_table_print_dmar_entry(struct acpi_dmar_header
*header
)
383 struct acpi_dmar_hardware_unit
*drhd
;
384 struct acpi_dmar_reserved_memory
*rmrr
;
385 struct acpi_dmar_atsr
*atsr
;
386 struct acpi_dmar_rhsa
*rhsa
;
388 switch (header
->type
) {
389 case ACPI_DMAR_TYPE_HARDWARE_UNIT
:
390 drhd
= container_of(header
, struct acpi_dmar_hardware_unit
,
392 printk (KERN_INFO PREFIX
393 "DRHD base: %#016Lx flags: %#x\n",
394 (unsigned long long)drhd
->address
, drhd
->flags
);
396 case ACPI_DMAR_TYPE_RESERVED_MEMORY
:
397 rmrr
= container_of(header
, struct acpi_dmar_reserved_memory
,
399 printk (KERN_INFO PREFIX
400 "RMRR base: %#016Lx end: %#016Lx\n",
401 (unsigned long long)rmrr
->base_address
,
402 (unsigned long long)rmrr
->end_address
);
404 case ACPI_DMAR_TYPE_ATSR
:
405 atsr
= container_of(header
, struct acpi_dmar_atsr
, header
);
406 printk(KERN_INFO PREFIX
"ATSR flags: %#x\n", atsr
->flags
);
408 case ACPI_DMAR_HARDWARE_AFFINITY
:
409 rhsa
= container_of(header
, struct acpi_dmar_rhsa
, header
);
410 printk(KERN_INFO PREFIX
"RHSA base: %#016Lx proximity domain: %#x\n",
411 (unsigned long long)rhsa
->base_address
,
412 rhsa
->proximity_domain
);
418 * dmar_table_detect - checks to see if the platform supports DMAR devices
420 static int __init
dmar_table_detect(void)
422 acpi_status status
= AE_OK
;
424 /* if we could find DMAR table, then there are DMAR devices */
425 status
= acpi_get_table_with_size(ACPI_SIG_DMAR
, 0,
426 (struct acpi_table_header
**)&dmar_tbl
,
429 if (ACPI_SUCCESS(status
) && !dmar_tbl
) {
430 printk (KERN_WARNING PREFIX
"Unable to map DMAR\n");
431 status
= AE_NOT_FOUND
;
434 return (ACPI_SUCCESS(status
) ? 1 : 0);
438 * parse_dmar_table - parses the DMA reporting table
441 parse_dmar_table(void)
443 struct acpi_table_dmar
*dmar
;
444 struct acpi_dmar_header
*entry_header
;
448 * Do it again, earlier dmar_tbl mapping could be mapped with
454 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
455 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
457 dmar_tbl
= tboot_get_dmar_table(dmar_tbl
);
459 dmar
= (struct acpi_table_dmar
*)dmar_tbl
;
463 if (dmar
->width
< PAGE_SHIFT
- 1) {
464 printk(KERN_WARNING PREFIX
"Invalid DMAR haw\n");
468 printk (KERN_INFO PREFIX
"Host address width %d\n",
471 entry_header
= (struct acpi_dmar_header
*)(dmar
+ 1);
472 while (((unsigned long)entry_header
) <
473 (((unsigned long)dmar
) + dmar_tbl
->length
)) {
474 /* Avoid looping forever on bad ACPI tables */
475 if (entry_header
->length
== 0) {
476 printk(KERN_WARNING PREFIX
477 "Invalid 0-length structure\n");
482 dmar_table_print_dmar_entry(entry_header
);
484 switch (entry_header
->type
) {
485 case ACPI_DMAR_TYPE_HARDWARE_UNIT
:
486 ret
= dmar_parse_one_drhd(entry_header
);
488 case ACPI_DMAR_TYPE_RESERVED_MEMORY
:
490 ret
= dmar_parse_one_rmrr(entry_header
);
493 case ACPI_DMAR_TYPE_ATSR
:
495 ret
= dmar_parse_one_atsr(entry_header
);
498 case ACPI_DMAR_HARDWARE_AFFINITY
:
499 #ifdef CONFIG_ACPI_NUMA
500 ret
= dmar_parse_one_rhsa(entry_header
);
504 printk(KERN_WARNING PREFIX
505 "Unknown DMAR structure type %d\n",
507 ret
= 0; /* for forward compatibility */
513 entry_header
= ((void *)entry_header
+ entry_header
->length
);
518 int dmar_pci_device_match(struct pci_dev
*devices
[], int cnt
,
524 for (index
= 0; index
< cnt
; index
++)
525 if (dev
== devices
[index
])
528 /* Check our parent */
529 dev
= dev
->bus
->self
;
535 struct dmar_drhd_unit
*
536 dmar_find_matched_drhd_unit(struct pci_dev
*dev
)
538 struct dmar_drhd_unit
*dmaru
= NULL
;
539 struct acpi_dmar_hardware_unit
*drhd
;
541 list_for_each_entry(dmaru
, &dmar_drhd_units
, list
) {
542 drhd
= container_of(dmaru
->hdr
,
543 struct acpi_dmar_hardware_unit
,
546 if (dmaru
->include_all
&&
547 drhd
->segment
== pci_domain_nr(dev
->bus
))
550 if (dmar_pci_device_match(dmaru
->devices
,
551 dmaru
->devices_cnt
, dev
))
558 int __init
dmar_dev_scope_init(void)
560 struct dmar_drhd_unit
*drhd
, *drhd_n
;
563 list_for_each_entry_safe(drhd
, drhd_n
, &dmar_drhd_units
, list
) {
564 ret
= dmar_parse_dev(drhd
);
571 struct dmar_rmrr_unit
*rmrr
, *rmrr_n
;
572 struct dmar_atsr_unit
*atsr
, *atsr_n
;
574 list_for_each_entry_safe(rmrr
, rmrr_n
, &dmar_rmrr_units
, list
) {
575 ret
= rmrr_parse_dev(rmrr
);
580 list_for_each_entry_safe(atsr
, atsr_n
, &dmar_atsr_units
, list
) {
581 ret
= atsr_parse_dev(atsr
);
592 int __init
dmar_table_init(void)
594 static int dmar_table_initialized
;
597 if (dmar_table_initialized
)
600 dmar_table_initialized
= 1;
602 ret
= parse_dmar_table();
605 printk(KERN_INFO PREFIX
"parse DMAR table failure.\n");
609 if (list_empty(&dmar_drhd_units
)) {
610 printk(KERN_INFO PREFIX
"No DMAR devices found\n");
615 if (list_empty(&dmar_rmrr_units
))
616 printk(KERN_INFO PREFIX
"No RMRR found\n");
618 if (list_empty(&dmar_atsr_units
))
619 printk(KERN_INFO PREFIX
"No ATSR found\n");
625 void __init
detect_intel_iommu(void)
629 ret
= dmar_table_detect();
632 #ifdef CONFIG_INTR_REMAP
633 struct acpi_table_dmar
*dmar
;
635 * for now we will disable dma-remapping when interrupt
636 * remapping is enabled.
637 * When support for queued invalidation for IOTLB invalidation
638 * is added, we will not need this any more.
640 dmar
= (struct acpi_table_dmar
*) dmar_tbl
;
641 if (ret
&& cpu_has_x2apic
&& dmar
->flags
& 0x1)
643 "Queued invalidation will be enabled to support "
644 "x2apic and Intr-remapping.\n");
647 if (ret
&& !no_iommu
&& !iommu_detected
&& !swiotlb
&&
652 early_acpi_os_unmap_memory(dmar_tbl
, dmar_tbl_size
);
657 int alloc_iommu(struct dmar_drhd_unit
*drhd
)
659 struct intel_iommu
*iommu
;
662 static int iommu_allocated
= 0;
666 iommu
= kzalloc(sizeof(*iommu
), GFP_KERNEL
);
670 iommu
->seq_id
= iommu_allocated
++;
671 sprintf (iommu
->name
, "dmar%d", iommu
->seq_id
);
673 iommu
->reg
= ioremap(drhd
->reg_base_addr
, VTD_PAGE_SIZE
);
675 printk(KERN_ERR
"IOMMU: can't map the region\n");
678 iommu
->cap
= dmar_readq(iommu
->reg
+ DMAR_CAP_REG
);
679 iommu
->ecap
= dmar_readq(iommu
->reg
+ DMAR_ECAP_REG
);
681 if (iommu
->cap
== (uint64_t)-1 && iommu
->ecap
== (uint64_t)-1) {
682 /* Promote an attitude of violence to a BIOS engineer today */
683 WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
684 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
686 dmi_get_system_info(DMI_BIOS_VENDOR
),
687 dmi_get_system_info(DMI_BIOS_VERSION
),
688 dmi_get_system_info(DMI_PRODUCT_VERSION
));
693 agaw
= iommu_calculate_agaw(iommu
);
696 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
700 msagaw
= iommu_calculate_max_sagaw(iommu
);
703 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
709 iommu
->msagaw
= msagaw
;
713 /* the registers might be more than one page */
714 map_size
= max_t(int, ecap_max_iotlb_offset(iommu
->ecap
),
715 cap_max_fault_reg_offset(iommu
->cap
));
716 map_size
= VTD_PAGE_ALIGN(map_size
);
717 if (map_size
> VTD_PAGE_SIZE
) {
719 iommu
->reg
= ioremap(drhd
->reg_base_addr
, map_size
);
721 printk(KERN_ERR
"IOMMU: can't map the region\n");
726 ver
= readl(iommu
->reg
+ DMAR_VER_REG
);
727 pr_info("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
728 (unsigned long long)drhd
->reg_base_addr
,
729 DMAR_VER_MAJOR(ver
), DMAR_VER_MINOR(ver
),
730 (unsigned long long)iommu
->cap
,
731 (unsigned long long)iommu
->ecap
);
733 spin_lock_init(&iommu
->register_lock
);
745 void free_iommu(struct intel_iommu
*iommu
)
751 free_dmar_iommu(iommu
);
760 * Reclaim all the submitted descriptors which have completed its work.
762 static inline void reclaim_free_desc(struct q_inval
*qi
)
764 while (qi
->desc_status
[qi
->free_tail
] == QI_DONE
||
765 qi
->desc_status
[qi
->free_tail
] == QI_ABORT
) {
766 qi
->desc_status
[qi
->free_tail
] = QI_FREE
;
767 qi
->free_tail
= (qi
->free_tail
+ 1) % QI_LENGTH
;
772 static int qi_check_fault(struct intel_iommu
*iommu
, int index
)
776 struct q_inval
*qi
= iommu
->qi
;
777 int wait_index
= (index
+ 1) % QI_LENGTH
;
779 if (qi
->desc_status
[wait_index
] == QI_ABORT
)
782 fault
= readl(iommu
->reg
+ DMAR_FSTS_REG
);
785 * If IQE happens, the head points to the descriptor associated
786 * with the error. No new descriptors are fetched until the IQE
789 if (fault
& DMA_FSTS_IQE
) {
790 head
= readl(iommu
->reg
+ DMAR_IQH_REG
);
791 if ((head
>> DMAR_IQ_SHIFT
) == index
) {
792 printk(KERN_ERR
"VT-d detected invalid descriptor: "
793 "low=%llx, high=%llx\n",
794 (unsigned long long)qi
->desc
[index
].low
,
795 (unsigned long long)qi
->desc
[index
].high
);
796 memcpy(&qi
->desc
[index
], &qi
->desc
[wait_index
],
797 sizeof(struct qi_desc
));
798 __iommu_flush_cache(iommu
, &qi
->desc
[index
],
799 sizeof(struct qi_desc
));
800 writel(DMA_FSTS_IQE
, iommu
->reg
+ DMAR_FSTS_REG
);
806 * If ITE happens, all pending wait_desc commands are aborted.
807 * No new descriptors are fetched until the ITE is cleared.
809 if (fault
& DMA_FSTS_ITE
) {
810 head
= readl(iommu
->reg
+ DMAR_IQH_REG
);
811 head
= ((head
>> DMAR_IQ_SHIFT
) - 1 + QI_LENGTH
) % QI_LENGTH
;
813 tail
= readl(iommu
->reg
+ DMAR_IQT_REG
);
814 tail
= ((tail
>> DMAR_IQ_SHIFT
) - 1 + QI_LENGTH
) % QI_LENGTH
;
816 writel(DMA_FSTS_ITE
, iommu
->reg
+ DMAR_FSTS_REG
);
819 if (qi
->desc_status
[head
] == QI_IN_USE
)
820 qi
->desc_status
[head
] = QI_ABORT
;
821 head
= (head
- 2 + QI_LENGTH
) % QI_LENGTH
;
822 } while (head
!= tail
);
824 if (qi
->desc_status
[wait_index
] == QI_ABORT
)
828 if (fault
& DMA_FSTS_ICE
)
829 writel(DMA_FSTS_ICE
, iommu
->reg
+ DMAR_FSTS_REG
);
835 * Submit the queued invalidation descriptor to the remapping
836 * hardware unit and wait for its completion.
838 int qi_submit_sync(struct qi_desc
*desc
, struct intel_iommu
*iommu
)
841 struct q_inval
*qi
= iommu
->qi
;
842 struct qi_desc
*hw
, wait_desc
;
843 int wait_index
, index
;
854 spin_lock_irqsave(&qi
->q_lock
, flags
);
855 while (qi
->free_cnt
< 3) {
856 spin_unlock_irqrestore(&qi
->q_lock
, flags
);
858 spin_lock_irqsave(&qi
->q_lock
, flags
);
861 index
= qi
->free_head
;
862 wait_index
= (index
+ 1) % QI_LENGTH
;
864 qi
->desc_status
[index
] = qi
->desc_status
[wait_index
] = QI_IN_USE
;
868 wait_desc
.low
= QI_IWD_STATUS_DATA(QI_DONE
) |
869 QI_IWD_STATUS_WRITE
| QI_IWD_TYPE
;
870 wait_desc
.high
= virt_to_phys(&qi
->desc_status
[wait_index
]);
872 hw
[wait_index
] = wait_desc
;
874 __iommu_flush_cache(iommu
, &hw
[index
], sizeof(struct qi_desc
));
875 __iommu_flush_cache(iommu
, &hw
[wait_index
], sizeof(struct qi_desc
));
877 qi
->free_head
= (qi
->free_head
+ 2) % QI_LENGTH
;
881 * update the HW tail register indicating the presence of
884 writel(qi
->free_head
<< DMAR_IQ_SHIFT
, iommu
->reg
+ DMAR_IQT_REG
);
886 while (qi
->desc_status
[wait_index
] != QI_DONE
) {
888 * We will leave the interrupts disabled, to prevent interrupt
889 * context to queue another cmd while a cmd is already submitted
890 * and waiting for completion on this cpu. This is to avoid
891 * a deadlock where the interrupt context can wait indefinitely
892 * for free slots in the queue.
894 rc
= qi_check_fault(iommu
, index
);
898 spin_unlock(&qi
->q_lock
);
900 spin_lock(&qi
->q_lock
);
903 qi
->desc_status
[index
] = QI_DONE
;
905 reclaim_free_desc(qi
);
906 spin_unlock_irqrestore(&qi
->q_lock
, flags
);
915 * Flush the global interrupt entry cache.
917 void qi_global_iec(struct intel_iommu
*iommu
)
921 desc
.low
= QI_IEC_TYPE
;
924 /* should never fail */
925 qi_submit_sync(&desc
, iommu
);
928 void qi_flush_context(struct intel_iommu
*iommu
, u16 did
, u16 sid
, u8 fm
,
933 desc
.low
= QI_CC_FM(fm
) | QI_CC_SID(sid
) | QI_CC_DID(did
)
934 | QI_CC_GRAN(type
) | QI_CC_TYPE
;
937 qi_submit_sync(&desc
, iommu
);
940 void qi_flush_iotlb(struct intel_iommu
*iommu
, u16 did
, u64 addr
,
941 unsigned int size_order
, u64 type
)
948 if (cap_write_drain(iommu
->cap
))
951 if (cap_read_drain(iommu
->cap
))
954 desc
.low
= QI_IOTLB_DID(did
) | QI_IOTLB_DR(dr
) | QI_IOTLB_DW(dw
)
955 | QI_IOTLB_GRAN(type
) | QI_IOTLB_TYPE
;
956 desc
.high
= QI_IOTLB_ADDR(addr
) | QI_IOTLB_IH(ih
)
957 | QI_IOTLB_AM(size_order
);
959 qi_submit_sync(&desc
, iommu
);
962 void qi_flush_dev_iotlb(struct intel_iommu
*iommu
, u16 sid
, u16 qdep
,
963 u64 addr
, unsigned mask
)
968 BUG_ON(addr
& ((1 << (VTD_PAGE_SHIFT
+ mask
)) - 1));
969 addr
|= (1 << (VTD_PAGE_SHIFT
+ mask
- 1)) - 1;
970 desc
.high
= QI_DEV_IOTLB_ADDR(addr
) | QI_DEV_IOTLB_SIZE
;
972 desc
.high
= QI_DEV_IOTLB_ADDR(addr
);
974 if (qdep
>= QI_DEV_IOTLB_MAX_INVS
)
977 desc
.low
= QI_DEV_IOTLB_SID(sid
) | QI_DEV_IOTLB_QDEP(qdep
) |
980 qi_submit_sync(&desc
, iommu
);
984 * Disable Queued Invalidation interface.
986 void dmar_disable_qi(struct intel_iommu
*iommu
)
990 cycles_t start_time
= get_cycles();
992 if (!ecap_qis(iommu
->ecap
))
995 spin_lock_irqsave(&iommu
->register_lock
, flags
);
997 sts
= dmar_readq(iommu
->reg
+ DMAR_GSTS_REG
);
998 if (!(sts
& DMA_GSTS_QIES
))
1002 * Give a chance to HW to complete the pending invalidation requests.
1004 while ((readl(iommu
->reg
+ DMAR_IQT_REG
) !=
1005 readl(iommu
->reg
+ DMAR_IQH_REG
)) &&
1006 (DMAR_OPERATION_TIMEOUT
> (get_cycles() - start_time
)))
1009 iommu
->gcmd
&= ~DMA_GCMD_QIE
;
1010 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1012 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
, readl
,
1013 !(sts
& DMA_GSTS_QIES
), sts
);
1015 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1019 * Enable queued invalidation.
1021 static void __dmar_enable_qi(struct intel_iommu
*iommu
)
1024 unsigned long flags
;
1025 struct q_inval
*qi
= iommu
->qi
;
1027 qi
->free_head
= qi
->free_tail
= 0;
1028 qi
->free_cnt
= QI_LENGTH
;
1030 spin_lock_irqsave(&iommu
->register_lock
, flags
);
1032 /* write zero to the tail reg */
1033 writel(0, iommu
->reg
+ DMAR_IQT_REG
);
1035 dmar_writeq(iommu
->reg
+ DMAR_IQA_REG
, virt_to_phys(qi
->desc
));
1037 iommu
->gcmd
|= DMA_GCMD_QIE
;
1038 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1040 /* Make sure hardware complete it */
1041 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
, readl
, (sts
& DMA_GSTS_QIES
), sts
);
1043 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1047 * Enable Queued Invalidation interface. This is a must to support
1048 * interrupt-remapping. Also used by DMA-remapping, which replaces
1049 * register based IOTLB invalidation.
1051 int dmar_enable_qi(struct intel_iommu
*iommu
)
1054 struct page
*desc_page
;
1056 if (!ecap_qis(iommu
->ecap
))
1060 * queued invalidation is already setup and enabled.
1065 iommu
->qi
= kmalloc(sizeof(*qi
), GFP_ATOMIC
);
1072 desc_page
= alloc_pages_node(iommu
->node
, GFP_ATOMIC
| __GFP_ZERO
, 0);
1079 qi
->desc
= page_address(desc_page
);
1081 qi
->desc_status
= kmalloc(QI_LENGTH
* sizeof(int), GFP_ATOMIC
);
1082 if (!qi
->desc_status
) {
1083 free_page((unsigned long) qi
->desc
);
1089 qi
->free_head
= qi
->free_tail
= 0;
1090 qi
->free_cnt
= QI_LENGTH
;
1092 spin_lock_init(&qi
->q_lock
);
1094 __dmar_enable_qi(iommu
);
1099 /* iommu interrupt handling. Most stuff are MSI-like. */
1107 static const char *dma_remap_fault_reasons
[] =
1110 "Present bit in root entry is clear",
1111 "Present bit in context entry is clear",
1112 "Invalid context entry",
1113 "Access beyond MGAW",
1114 "PTE Write access is not set",
1115 "PTE Read access is not set",
1116 "Next page table ptr is invalid",
1117 "Root table address invalid",
1118 "Context table ptr is invalid",
1119 "non-zero reserved fields in RTP",
1120 "non-zero reserved fields in CTP",
1121 "non-zero reserved fields in PTE",
1124 static const char *intr_remap_fault_reasons
[] =
1126 "Detected reserved fields in the decoded interrupt-remapped request",
1127 "Interrupt index exceeded the interrupt-remapping table size",
1128 "Present field in the IRTE entry is clear",
1129 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1130 "Detected reserved fields in the IRTE entry",
1131 "Blocked a compatibility format interrupt request",
1132 "Blocked an interrupt request due to source-id verification failure",
1135 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1137 const char *dmar_get_fault_reason(u8 fault_reason
, int *fault_type
)
1139 if (fault_reason
>= 0x20 && (fault_reason
<= 0x20 +
1140 ARRAY_SIZE(intr_remap_fault_reasons
))) {
1141 *fault_type
= INTR_REMAP
;
1142 return intr_remap_fault_reasons
[fault_reason
- 0x20];
1143 } else if (fault_reason
< ARRAY_SIZE(dma_remap_fault_reasons
)) {
1144 *fault_type
= DMA_REMAP
;
1145 return dma_remap_fault_reasons
[fault_reason
];
1147 *fault_type
= UNKNOWN
;
1152 void dmar_msi_unmask(unsigned int irq
)
1154 struct intel_iommu
*iommu
= get_irq_data(irq
);
1158 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1159 writel(0, iommu
->reg
+ DMAR_FECTL_REG
);
1160 /* Read a reg to force flush the post write */
1161 readl(iommu
->reg
+ DMAR_FECTL_REG
);
1162 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1165 void dmar_msi_mask(unsigned int irq
)
1168 struct intel_iommu
*iommu
= get_irq_data(irq
);
1171 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1172 writel(DMA_FECTL_IM
, iommu
->reg
+ DMAR_FECTL_REG
);
1173 /* Read a reg to force flush the post write */
1174 readl(iommu
->reg
+ DMAR_FECTL_REG
);
1175 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1178 void dmar_msi_write(int irq
, struct msi_msg
*msg
)
1180 struct intel_iommu
*iommu
= get_irq_data(irq
);
1183 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1184 writel(msg
->data
, iommu
->reg
+ DMAR_FEDATA_REG
);
1185 writel(msg
->address_lo
, iommu
->reg
+ DMAR_FEADDR_REG
);
1186 writel(msg
->address_hi
, iommu
->reg
+ DMAR_FEUADDR_REG
);
1187 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1190 void dmar_msi_read(int irq
, struct msi_msg
*msg
)
1192 struct intel_iommu
*iommu
= get_irq_data(irq
);
1195 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1196 msg
->data
= readl(iommu
->reg
+ DMAR_FEDATA_REG
);
1197 msg
->address_lo
= readl(iommu
->reg
+ DMAR_FEADDR_REG
);
1198 msg
->address_hi
= readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
1199 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1202 static int dmar_fault_do_one(struct intel_iommu
*iommu
, int type
,
1203 u8 fault_reason
, u16 source_id
, unsigned long long addr
)
1208 reason
= dmar_get_fault_reason(fault_reason
, &fault_type
);
1210 if (fault_type
== INTR_REMAP
)
1211 printk(KERN_ERR
"INTR-REMAP: Request device [[%02x:%02x.%d] "
1212 "fault index %llx\n"
1213 "INTR-REMAP:[fault reason %02d] %s\n",
1214 (source_id
>> 8), PCI_SLOT(source_id
& 0xFF),
1215 PCI_FUNC(source_id
& 0xFF), addr
>> 48,
1216 fault_reason
, reason
);
1219 "DMAR:[%s] Request device [%02x:%02x.%d] "
1220 "fault addr %llx \n"
1221 "DMAR:[fault reason %02d] %s\n",
1222 (type
? "DMA Read" : "DMA Write"),
1223 (source_id
>> 8), PCI_SLOT(source_id
& 0xFF),
1224 PCI_FUNC(source_id
& 0xFF), addr
, fault_reason
, reason
);
1228 #define PRIMARY_FAULT_REG_LEN (16)
1229 irqreturn_t
dmar_fault(int irq
, void *dev_id
)
1231 struct intel_iommu
*iommu
= dev_id
;
1232 int reg
, fault_index
;
1236 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1237 fault_status
= readl(iommu
->reg
+ DMAR_FSTS_REG
);
1239 printk(KERN_ERR
"DRHD: handling fault status reg %x\n",
1242 /* TBD: ignore advanced fault log currently */
1243 if (!(fault_status
& DMA_FSTS_PPF
))
1246 fault_index
= dma_fsts_fault_record_index(fault_status
);
1247 reg
= cap_fault_reg_offset(iommu
->cap
);
1255 /* highest 32 bits */
1256 data
= readl(iommu
->reg
+ reg
+
1257 fault_index
* PRIMARY_FAULT_REG_LEN
+ 12);
1258 if (!(data
& DMA_FRCD_F
))
1261 fault_reason
= dma_frcd_fault_reason(data
);
1262 type
= dma_frcd_type(data
);
1264 data
= readl(iommu
->reg
+ reg
+
1265 fault_index
* PRIMARY_FAULT_REG_LEN
+ 8);
1266 source_id
= dma_frcd_source_id(data
);
1268 guest_addr
= dmar_readq(iommu
->reg
+ reg
+
1269 fault_index
* PRIMARY_FAULT_REG_LEN
);
1270 guest_addr
= dma_frcd_page_addr(guest_addr
);
1271 /* clear the fault */
1272 writel(DMA_FRCD_F
, iommu
->reg
+ reg
+
1273 fault_index
* PRIMARY_FAULT_REG_LEN
+ 12);
1275 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1277 dmar_fault_do_one(iommu
, type
, fault_reason
,
1278 source_id
, guest_addr
);
1281 if (fault_index
>= cap_num_fault_regs(iommu
->cap
))
1283 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1286 /* clear all the other faults */
1287 fault_status
= readl(iommu
->reg
+ DMAR_FSTS_REG
);
1288 writel(fault_status
, iommu
->reg
+ DMAR_FSTS_REG
);
1290 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1294 int dmar_set_interrupt(struct intel_iommu
*iommu
)
1299 * Check if the fault interrupt is already initialized.
1306 printk(KERN_ERR
"IOMMU: no free vectors\n");
1310 set_irq_data(irq
, iommu
);
1313 ret
= arch_setup_dmar_msi(irq
);
1315 set_irq_data(irq
, NULL
);
1321 ret
= request_irq(irq
, dmar_fault
, 0, iommu
->name
, iommu
);
1323 printk(KERN_ERR
"IOMMU: can't request irq\n");
1327 int __init
enable_drhd_fault_handling(void)
1329 struct dmar_drhd_unit
*drhd
;
1332 * Enable fault control interrupt.
1334 for_each_drhd_unit(drhd
) {
1336 struct intel_iommu
*iommu
= drhd
->iommu
;
1337 ret
= dmar_set_interrupt(iommu
);
1340 printk(KERN_ERR
"DRHD %Lx: failed to enable fault, "
1341 " interrupt, ret %d\n",
1342 (unsigned long long)drhd
->reg_base_addr
, ret
);
1351 * Re-enable Queued Invalidation interface.
1353 int dmar_reenable_qi(struct intel_iommu
*iommu
)
1355 if (!ecap_qis(iommu
->ecap
))
1362 * First disable queued invalidation.
1364 dmar_disable_qi(iommu
);
1366 * Then enable queued invalidation again. Since there is no pending
1367 * invalidation requests now, it's safe to re-enable queued
1370 __dmar_enable_qi(iommu
);
1376 * Check interrupt remapping support in DMAR table description.
1378 int dmar_ir_support(void)
1380 struct acpi_table_dmar
*dmar
;
1381 dmar
= (struct acpi_table_dmar
*)dmar_tbl
;
1382 return dmar
->flags
& 0x1;