1 #include <linux/interrupt.h>
2 #include <linux/dmar.h>
3 #include <linux/spinlock.h>
4 #include <linux/slab.h>
5 #include <linux/jiffies.h>
6 #include <linux/hpet.h>
9 #include <asm/io_apic.h>
12 #include <linux/intel-iommu.h>
13 #include "intr_remapping.h"
14 #include <acpi/acpi.h>
15 #include <asm/pci-direct.h>
18 static struct ioapic_scope ir_ioapic
[MAX_IO_APICS
];
19 static struct hpet_scope ir_hpet
[MAX_HPET_TBS
];
20 static int ir_ioapic_num
, ir_hpet_num
;
21 int intr_remapping_enabled
;
23 static int disable_intremap
;
24 static int disable_sourceid_checking
;
26 static __init
int setup_nointremap(char *str
)
31 early_param("nointremap", setup_nointremap
);
33 static __init
int setup_intremap(char *str
)
38 if (!strncmp(str
, "on", 2))
40 else if (!strncmp(str
, "off", 3))
42 else if (!strncmp(str
, "nosid", 5))
43 disable_sourceid_checking
= 1;
47 early_param("intremap", setup_intremap
);
50 struct intel_iommu
*iommu
;
56 #ifdef CONFIG_GENERIC_HARDIRQS
57 static struct irq_2_iommu
*irq_2_iommu(unsigned int irq
)
59 return get_irq_iommu(irq
);
62 static struct irq_2_iommu
*irq_2_iommu_alloc(unsigned int irq
)
64 struct irq_data
*data
= irq_get_irq_data(irq
);
66 if (WARN_ONCE(data
->irq_2_iommu
,
67 KERN_DEBUG
"irq_2_iommu!=NULL irq %u\n", irq
))
68 return data
->irq_2_iommu
;
70 data
->irq_2_iommu
= kzalloc_node(sizeof(*data
->irq_2_iommu
),
71 GFP_ATOMIC
, data
->node
);
72 return data
->irq_2_iommu
;
75 static void irq_2_iommu_free(unsigned int irq
)
77 struct irq_data
*d
= irq_get_irq_data(irq
);
78 struct irq_2_iommu
*p
= d
->irq_2_iommu
;
80 d
->irq_2_iommu
= NULL
;
84 #else /* !CONFIG_SPARSE_IRQ */
86 static struct irq_2_iommu irq_2_iommuX
[NR_IRQS
];
88 static struct irq_2_iommu
*irq_2_iommu(unsigned int irq
)
91 return &irq_2_iommuX
[irq
];
95 static struct irq_2_iommu
*irq_2_iommu_alloc(unsigned int irq
)
97 return irq_2_iommu(irq
);
100 static void irq_2_iommu_free(unsigned int irq
) { }
104 static DEFINE_SPINLOCK(irq_2_ir_lock
);
106 static struct irq_2_iommu
*valid_irq_2_iommu(unsigned int irq
)
108 struct irq_2_iommu
*irq_iommu
;
110 irq_iommu
= irq_2_iommu(irq
);
115 if (!irq_iommu
->iommu
)
121 int irq_remapped(int irq
)
123 return valid_irq_2_iommu(irq
) != NULL
;
126 int get_irte(int irq
, struct irte
*entry
)
129 struct irq_2_iommu
*irq_iommu
;
135 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
136 irq_iommu
= valid_irq_2_iommu(irq
);
138 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
142 index
= irq_iommu
->irte_index
+ irq_iommu
->sub_handle
;
143 *entry
= *(irq_iommu
->iommu
->ir_table
->base
+ index
);
145 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
149 int alloc_irte(struct intel_iommu
*iommu
, int irq
, u16 count
)
151 struct ir_table
*table
= iommu
->ir_table
;
152 struct irq_2_iommu
*irq_iommu
;
153 u16 index
, start_index
;
154 unsigned int mask
= 0;
161 #ifndef CONFIG_SPARSE_IRQ
162 /* protect irq_2_iommu_alloc later */
168 * start the IRTE search from index 0.
170 index
= start_index
= 0;
173 count
= __roundup_pow_of_two(count
);
177 if (mask
> ecap_max_handle_mask(iommu
->ecap
)) {
179 "Requested mask %x exceeds the max invalidation handle"
180 " mask value %Lx\n", mask
,
181 ecap_max_handle_mask(iommu
->ecap
));
185 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
187 for (i
= index
; i
< index
+ count
; i
++)
188 if (table
->base
[i
].present
)
190 /* empty index found */
191 if (i
== index
+ count
)
194 index
= (index
+ count
) % INTR_REMAP_TABLE_ENTRIES
;
196 if (index
== start_index
) {
197 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
198 printk(KERN_ERR
"can't allocate an IRTE\n");
203 for (i
= index
; i
< index
+ count
; i
++)
204 table
->base
[i
].present
= 1;
206 irq_iommu
= irq_2_iommu_alloc(irq
);
208 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
209 printk(KERN_ERR
"can't allocate irq_2_iommu\n");
213 irq_iommu
->iommu
= iommu
;
214 irq_iommu
->irte_index
= index
;
215 irq_iommu
->sub_handle
= 0;
216 irq_iommu
->irte_mask
= mask
;
218 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
223 static int qi_flush_iec(struct intel_iommu
*iommu
, int index
, int mask
)
227 desc
.low
= QI_IEC_IIDEX(index
) | QI_IEC_TYPE
| QI_IEC_IM(mask
)
231 return qi_submit_sync(&desc
, iommu
);
234 int map_irq_to_irte_handle(int irq
, u16
*sub_handle
)
237 struct irq_2_iommu
*irq_iommu
;
240 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
241 irq_iommu
= valid_irq_2_iommu(irq
);
243 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
247 *sub_handle
= irq_iommu
->sub_handle
;
248 index
= irq_iommu
->irte_index
;
249 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
253 int set_irte_irq(int irq
, struct intel_iommu
*iommu
, u16 index
, u16 subhandle
)
255 struct irq_2_iommu
*irq_iommu
;
258 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
260 irq_iommu
= irq_2_iommu_alloc(irq
);
263 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
264 printk(KERN_ERR
"can't allocate irq_2_iommu\n");
268 irq_iommu
->iommu
= iommu
;
269 irq_iommu
->irte_index
= index
;
270 irq_iommu
->sub_handle
= subhandle
;
271 irq_iommu
->irte_mask
= 0;
273 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
278 int modify_irte(int irq
, struct irte
*irte_modified
)
283 struct intel_iommu
*iommu
;
284 struct irq_2_iommu
*irq_iommu
;
287 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
288 irq_iommu
= valid_irq_2_iommu(irq
);
290 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
294 iommu
= irq_iommu
->iommu
;
296 index
= irq_iommu
->irte_index
+ irq_iommu
->sub_handle
;
297 irte
= &iommu
->ir_table
->base
[index
];
299 set_64bit(&irte
->low
, irte_modified
->low
);
300 set_64bit(&irte
->high
, irte_modified
->high
);
301 __iommu_flush_cache(iommu
, irte
, sizeof(*irte
));
303 rc
= qi_flush_iec(iommu
, index
, 0);
304 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
309 struct intel_iommu
*map_hpet_to_ir(u8 hpet_id
)
313 for (i
= 0; i
< MAX_HPET_TBS
; i
++)
314 if (ir_hpet
[i
].id
== hpet_id
)
315 return ir_hpet
[i
].iommu
;
319 struct intel_iommu
*map_ioapic_to_ir(int apic
)
323 for (i
= 0; i
< MAX_IO_APICS
; i
++)
324 if (ir_ioapic
[i
].id
== apic
)
325 return ir_ioapic
[i
].iommu
;
329 struct intel_iommu
*map_dev_to_ir(struct pci_dev
*dev
)
331 struct dmar_drhd_unit
*drhd
;
333 drhd
= dmar_find_matched_drhd_unit(dev
);
340 static int clear_entries(struct irq_2_iommu
*irq_iommu
)
342 struct irte
*start
, *entry
, *end
;
343 struct intel_iommu
*iommu
;
346 if (irq_iommu
->sub_handle
)
349 iommu
= irq_iommu
->iommu
;
350 index
= irq_iommu
->irte_index
+ irq_iommu
->sub_handle
;
352 start
= iommu
->ir_table
->base
+ index
;
353 end
= start
+ (1 << irq_iommu
->irte_mask
);
355 for (entry
= start
; entry
< end
; entry
++) {
356 set_64bit(&entry
->low
, 0);
357 set_64bit(&entry
->high
, 0);
360 return qi_flush_iec(iommu
, index
, irq_iommu
->irte_mask
);
363 int free_irte(int irq
)
366 struct irq_2_iommu
*irq_iommu
;
369 spin_lock_irqsave(&irq_2_ir_lock
, flags
);
370 irq_iommu
= valid_irq_2_iommu(irq
);
372 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
376 rc
= clear_entries(irq_iommu
);
378 irq_iommu
->iommu
= NULL
;
379 irq_iommu
->irte_index
= 0;
380 irq_iommu
->sub_handle
= 0;
381 irq_iommu
->irte_mask
= 0;
383 spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
385 irq_2_iommu_free(irq
);
391 * source validation type
393 #define SVT_NO_VERIFY 0x0 /* no verification is required */
394 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fiels */
395 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
398 * source-id qualifier
400 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
401 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
402 * the third least significant bit
404 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
405 * the second and third least significant bits
407 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
408 * the least three significant bits
412 * set SVT, SQ and SID fields of irte to verify
413 * source ids of interrupt requests
415 static void set_irte_sid(struct irte
*irte
, unsigned int svt
,
416 unsigned int sq
, unsigned int sid
)
418 if (disable_sourceid_checking
)
425 int set_ioapic_sid(struct irte
*irte
, int apic
)
433 for (i
= 0; i
< MAX_IO_APICS
; i
++) {
434 if (ir_ioapic
[i
].id
== apic
) {
435 sid
= (ir_ioapic
[i
].bus
<< 8) | ir_ioapic
[i
].devfn
;
441 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic
);
445 set_irte_sid(irte
, 1, 0, sid
);
450 int set_hpet_sid(struct irte
*irte
, u8 id
)
458 for (i
= 0; i
< MAX_HPET_TBS
; i
++) {
459 if (ir_hpet
[i
].id
== id
) {
460 sid
= (ir_hpet
[i
].bus
<< 8) | ir_hpet
[i
].devfn
;
466 pr_warning("Failed to set source-id of HPET block (%d)\n", id
);
471 * Should really use SQ_ALL_16. Some platforms are broken.
472 * While we figure out the right quirks for these broken platforms, use
473 * SQ_13_IGNORE_3 for now.
475 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_13_IGNORE_3
, sid
);
480 int set_msi_sid(struct irte
*irte
, struct pci_dev
*dev
)
482 struct pci_dev
*bridge
;
487 /* PCIe device or Root Complex integrated PCI device */
488 if (pci_is_pcie(dev
) || !dev
->bus
->parent
) {
489 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
,
490 (dev
->bus
->number
<< 8) | dev
->devfn
);
494 bridge
= pci_find_upstream_pcie_bridge(dev
);
496 if (pci_is_pcie(bridge
))/* this is a PCIe-to-PCI/PCIX bridge */
497 set_irte_sid(irte
, SVT_VERIFY_BUS
, SQ_ALL_16
,
498 (bridge
->bus
->number
<< 8) | dev
->bus
->number
);
499 else /* this is a legacy PCI bridge */
500 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
,
501 (bridge
->bus
->number
<< 8) | bridge
->devfn
);
507 static void iommu_set_intr_remapping(struct intel_iommu
*iommu
, int mode
)
513 addr
= virt_to_phys((void *)iommu
->ir_table
->base
);
515 spin_lock_irqsave(&iommu
->register_lock
, flags
);
517 dmar_writeq(iommu
->reg
+ DMAR_IRTA_REG
,
518 (addr
) | IR_X2APIC_MODE(mode
) | INTR_REMAP_TABLE_REG_SIZE
);
520 /* Set interrupt-remapping table pointer */
521 iommu
->gcmd
|= DMA_GCMD_SIRTP
;
522 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
524 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
525 readl
, (sts
& DMA_GSTS_IRTPS
), sts
);
526 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
529 * global invalidation of interrupt entry cache before enabling
530 * interrupt-remapping.
532 qi_global_iec(iommu
);
534 spin_lock_irqsave(&iommu
->register_lock
, flags
);
536 /* Enable interrupt-remapping */
537 iommu
->gcmd
|= DMA_GCMD_IRE
;
538 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
540 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
541 readl
, (sts
& DMA_GSTS_IRES
), sts
);
543 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
547 static int setup_intr_remapping(struct intel_iommu
*iommu
, int mode
)
549 struct ir_table
*ir_table
;
552 ir_table
= iommu
->ir_table
= kzalloc(sizeof(struct ir_table
),
555 if (!iommu
->ir_table
)
558 pages
= alloc_pages_node(iommu
->node
, GFP_ATOMIC
| __GFP_ZERO
,
559 INTR_REMAP_PAGE_ORDER
);
562 printk(KERN_ERR
"failed to allocate pages of order %d\n",
563 INTR_REMAP_PAGE_ORDER
);
564 kfree(iommu
->ir_table
);
568 ir_table
->base
= page_address(pages
);
570 iommu_set_intr_remapping(iommu
, mode
);
575 * Disable Interrupt Remapping.
577 static void iommu_disable_intr_remapping(struct intel_iommu
*iommu
)
582 if (!ecap_ir_support(iommu
->ecap
))
586 * global invalidation of interrupt entry cache before disabling
587 * interrupt-remapping.
589 qi_global_iec(iommu
);
591 spin_lock_irqsave(&iommu
->register_lock
, flags
);
593 sts
= dmar_readq(iommu
->reg
+ DMAR_GSTS_REG
);
594 if (!(sts
& DMA_GSTS_IRES
))
597 iommu
->gcmd
&= ~DMA_GCMD_IRE
;
598 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
600 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
601 readl
, !(sts
& DMA_GSTS_IRES
), sts
);
604 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
607 int __init
intr_remapping_supported(void)
609 struct dmar_drhd_unit
*drhd
;
611 if (disable_intremap
)
614 if (!dmar_ir_support())
617 for_each_drhd_unit(drhd
) {
618 struct intel_iommu
*iommu
= drhd
->iommu
;
620 if (!ecap_ir_support(iommu
->ecap
))
627 int __init
enable_intr_remapping(int eim
)
629 struct dmar_drhd_unit
*drhd
;
632 if (parse_ioapics_under_ir() != 1) {
633 printk(KERN_INFO
"Not enable interrupt remapping\n");
637 for_each_drhd_unit(drhd
) {
638 struct intel_iommu
*iommu
= drhd
->iommu
;
641 * If the queued invalidation is already initialized,
642 * shouldn't disable it.
648 * Clear previous faults.
650 dmar_fault(-1, iommu
);
653 * Disable intr remapping and queued invalidation, if already
654 * enabled prior to OS handover.
656 iommu_disable_intr_remapping(iommu
);
658 dmar_disable_qi(iommu
);
662 * check for the Interrupt-remapping support
664 for_each_drhd_unit(drhd
) {
665 struct intel_iommu
*iommu
= drhd
->iommu
;
667 if (!ecap_ir_support(iommu
->ecap
))
670 if (eim
&& !ecap_eim_support(iommu
->ecap
)) {
671 printk(KERN_INFO
"DRHD %Lx: EIM not supported by DRHD, "
672 " ecap %Lx\n", drhd
->reg_base_addr
, iommu
->ecap
);
678 * Enable queued invalidation for all the DRHD's.
680 for_each_drhd_unit(drhd
) {
682 struct intel_iommu
*iommu
= drhd
->iommu
;
683 ret
= dmar_enable_qi(iommu
);
686 printk(KERN_ERR
"DRHD %Lx: failed to enable queued, "
687 " invalidation, ecap %Lx, ret %d\n",
688 drhd
->reg_base_addr
, iommu
->ecap
, ret
);
694 * Setup Interrupt-remapping for all the DRHD's now.
696 for_each_drhd_unit(drhd
) {
697 struct intel_iommu
*iommu
= drhd
->iommu
;
699 if (!ecap_ir_support(iommu
->ecap
))
702 if (setup_intr_remapping(iommu
, eim
))
711 intr_remapping_enabled
= 1;
717 * handle error condition gracefully here!
722 static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope
*scope
,
723 struct intel_iommu
*iommu
)
725 struct acpi_dmar_pci_path
*path
;
730 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
731 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
732 / sizeof(struct acpi_dmar_pci_path
);
734 while (--count
> 0) {
736 * Access PCI directly due to the PCI
737 * subsystem isn't initialized yet.
739 bus
= read_pci_config_byte(bus
, path
->dev
, path
->fn
,
743 ir_hpet
[ir_hpet_num
].bus
= bus
;
744 ir_hpet
[ir_hpet_num
].devfn
= PCI_DEVFN(path
->dev
, path
->fn
);
745 ir_hpet
[ir_hpet_num
].iommu
= iommu
;
746 ir_hpet
[ir_hpet_num
].id
= scope
->enumeration_id
;
750 static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope
*scope
,
751 struct intel_iommu
*iommu
)
753 struct acpi_dmar_pci_path
*path
;
758 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
759 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
760 / sizeof(struct acpi_dmar_pci_path
);
762 while (--count
> 0) {
764 * Access PCI directly due to the PCI
765 * subsystem isn't initialized yet.
767 bus
= read_pci_config_byte(bus
, path
->dev
, path
->fn
,
772 ir_ioapic
[ir_ioapic_num
].bus
= bus
;
773 ir_ioapic
[ir_ioapic_num
].devfn
= PCI_DEVFN(path
->dev
, path
->fn
);
774 ir_ioapic
[ir_ioapic_num
].iommu
= iommu
;
775 ir_ioapic
[ir_ioapic_num
].id
= scope
->enumeration_id
;
779 static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header
*header
,
780 struct intel_iommu
*iommu
)
782 struct acpi_dmar_hardware_unit
*drhd
;
783 struct acpi_dmar_device_scope
*scope
;
786 drhd
= (struct acpi_dmar_hardware_unit
*)header
;
788 start
= (void *)(drhd
+ 1);
789 end
= ((void *)drhd
) + header
->length
;
791 while (start
< end
) {
793 if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_IOAPIC
) {
794 if (ir_ioapic_num
== MAX_IO_APICS
) {
795 printk(KERN_WARNING
"Exceeded Max IO APICS\n");
799 printk(KERN_INFO
"IOAPIC id %d under DRHD base "
800 " 0x%Lx IOMMU %d\n", scope
->enumeration_id
,
801 drhd
->address
, iommu
->seq_id
);
803 ir_parse_one_ioapic_scope(scope
, iommu
);
804 } else if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_HPET
) {
805 if (ir_hpet_num
== MAX_HPET_TBS
) {
806 printk(KERN_WARNING
"Exceeded Max HPET blocks\n");
810 printk(KERN_INFO
"HPET id %d under DRHD base"
811 " 0x%Lx\n", scope
->enumeration_id
,
814 ir_parse_one_hpet_scope(scope
, iommu
);
816 start
+= scope
->length
;
823 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
826 int __init
parse_ioapics_under_ir(void)
828 struct dmar_drhd_unit
*drhd
;
829 int ir_supported
= 0;
831 for_each_drhd_unit(drhd
) {
832 struct intel_iommu
*iommu
= drhd
->iommu
;
834 if (ecap_ir_support(iommu
->ecap
)) {
835 if (ir_parse_ioapic_hpet_scope(drhd
->hdr
, iommu
))
842 if (ir_supported
&& ir_ioapic_num
!= nr_ioapics
) {
844 "Not all IO-APIC's listed under remapping hardware\n");
851 void disable_intr_remapping(void)
853 struct dmar_drhd_unit
*drhd
;
854 struct intel_iommu
*iommu
= NULL
;
857 * Disable Interrupt-remapping for all the DRHD's now.
859 for_each_iommu(iommu
, drhd
) {
860 if (!ecap_ir_support(iommu
->ecap
))
863 iommu_disable_intr_remapping(iommu
);
867 int reenable_intr_remapping(int eim
)
869 struct dmar_drhd_unit
*drhd
;
871 struct intel_iommu
*iommu
= NULL
;
873 for_each_iommu(iommu
, drhd
)
875 dmar_reenable_qi(iommu
);
878 * Setup Interrupt-remapping for all the DRHD's now.
880 for_each_iommu(iommu
, drhd
) {
881 if (!ecap_ir_support(iommu
->ecap
))
884 /* Set up interrupt remapping for iommu.*/
885 iommu_set_intr_remapping(iommu
, eim
);
896 * handle error condition gracefully here!