drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <asm/irq_remapping.h>
  42 #include <asm/iommu_table.h>
  43
  44 #include "irq_remapping.h"
  45
  46 /* No locks are needed as DMA remapping hardware unit
  47  * list is constructed at boot time and hotplug of
  48  * these units are not supported by the architecture.
  49  */
  50 LIST_HEAD(dmar_drhd_units);
  51
  52 struct acpi_table_header * __initdata dmar_tbl;
  53 static acpi_size dmar_tbl_size;
  54
  55 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  56 {
  57         /*
  58          * add INCLUDE_ALL at the tail, so scan the list will find it at
  59          * the very end.
  60          */
  61         if (drhd->include_all)
  62                 list_add_tail(&drhd->list, &dmar_drhd_units);
  63         else
  64                 list_add(&drhd->list, &dmar_drhd_units);
  65 }
  66
  67 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  68                                            struct pci_dev **dev, u16 segment)
  69 {
  70         struct pci_bus *bus;
  71         struct pci_dev *pdev = NULL;
  72         struct acpi_dmar_pci_path *path;
  73         int count;
  74
  75         bus = pci_find_bus(segment, scope->bus);
  76         path = (struct acpi_dmar_pci_path *)(scope + 1);
  77         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  78                 / sizeof(struct acpi_dmar_pci_path);
  79
  80         while (count) {
  81                 if (pdev)
  82                         pci_dev_put(pdev);
  83                 /*
  84                  * Some BIOSes list non-exist devices in DMAR table, just
  85                  * ignore it
  86                  */
  87                 if (!bus) {
  88                         pr_warn("Device scope bus [%d] not found\n", scope->bus);
  89                         break;
  90                 }
  91                 pdev = pci_get_slot(bus, PCI_DEVFN(path->device, path->function));
  92                 if (!pdev) {
  93                         /* warning will be printed below */
  94                         break;
  95                 }
  96                 path ++;
  97                 count --;
  98                 bus = pdev->subordinate;
  99         }
 100         if (!pdev) {
 101                 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                         segment, scope->bus, path->device, path->function);
 103                 return 0;
 104         }
 105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 106                         pdev->subordinate) || (scope->entry_type == \
 107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 108                 pci_dev_put(pdev);
 109                 pr_warn("Device scope type does not match for %s\n",
 110                         pci_name(pdev));
 111                 return -EINVAL;
 112         }
 113         *dev = pdev;
 114         return 0;
 115 }
 116
 117 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 118                                 struct pci_dev ***devices, u16 segment)
 119 {
 120         struct acpi_dmar_device_scope *scope;
 121         void * tmp = start;
 122         int index;
 123         int ret;
 124
 125         *cnt = 0;
 126         while (start < end) {
 127                 scope = start;
 128                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 129                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 130                         (*cnt)++;
 131                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 132                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 133                         pr_warn("Unsupported device scope\n");
 134                 }
 135                 start += scope->length;
 136         }
 137         if (*cnt == 0)
 138                 return 0;
 139
 140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 141         if (!*devices)
 142                 return -ENOMEM;
 143
 144         start = tmp;
 145         index = 0;
 146         while (start < end) {
 147                 scope = start;
 148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 150                         ret = dmar_parse_one_dev_scope(scope,
 151                                 &(*devices)[index], segment);
 152                         if (ret) {
 153                                 dmar_free_dev_scope(devices, cnt);
 154                                 return ret;
 155                         }
 156                         index ++;
 157                 }
 158                 start += scope->length;
 159         }
 160
 161         return 0;
 162 }
 163
 164 void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt)
 165 {
 166         if (*devices && *cnt) {
 167                 while (--*cnt >= 0)
 168                         pci_dev_put((*devices)[*cnt]);
 169                 kfree(*devices);
 170                 *devices = NULL;
 171                 *cnt = 0;
 172         }
 173 }
 174
 175 /**
 176  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 177  * structure which uniquely represent one DMA remapping hardware unit
 178  * present in the platform
 179  */
 180 static int __init
 181 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 182 {
 183         struct acpi_dmar_hardware_unit *drhd;
 184         struct dmar_drhd_unit *dmaru;
 185         int ret = 0;
 186
 187         drhd = (struct acpi_dmar_hardware_unit *)header;
 188         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 189         if (!dmaru)
 190                 return -ENOMEM;
 191
 192         dmaru->hdr = header;
 193         dmaru->reg_base_addr = drhd->address;
 194         dmaru->segment = drhd->segment;
 195         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 196
 197         ret = alloc_iommu(dmaru);
 198         if (ret) {
 199                 kfree(dmaru);
 200                 return ret;
 201         }
 202         dmar_register_drhd_unit(dmaru);
 203         return 0;
 204 }
 205
 206 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 207 {
 208         struct acpi_dmar_hardware_unit *drhd;
 209         int ret = 0;
 210
 211         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 212
 213         if (dmaru->include_all)
 214                 return 0;
 215
 216         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 217                                 ((void *)drhd) + drhd->header.length,
 218                                 &dmaru->devices_cnt, &dmaru->devices,
 219                                 drhd->segment);
 220         if (ret) {
 221                 list_del(&dmaru->list);
 222                 kfree(dmaru);
 223         }
 224         return ret;
 225 }
 226
 227 #ifdef CONFIG_ACPI_NUMA
 228 static int __init
 229 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 230 {
 231         struct acpi_dmar_rhsa *rhsa;
 232         struct dmar_drhd_unit *drhd;
 233
 234         rhsa = (struct acpi_dmar_rhsa *)header;
 235         for_each_drhd_unit(drhd) {
 236                 if (drhd->reg_base_addr == rhsa->base_address) {
 237                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 238
 239                         if (!node_online(node))
 240                                 node = -1;
 241                         drhd->iommu->node = node;
 242                         return 0;
 243                 }
 244         }
 245         WARN_TAINT(
 246                 1, TAINT_FIRMWARE_WORKAROUND,
 247                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 248                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 249                 drhd->reg_base_addr,
 250                 dmi_get_system_info(DMI_BIOS_VENDOR),
 251                 dmi_get_system_info(DMI_BIOS_VERSION),
 252                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 253
 254         return 0;
 255 }
 256 #endif
 257
 258 static void __init
 259 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 260 {
 261         struct acpi_dmar_hardware_unit *drhd;
 262         struct acpi_dmar_reserved_memory *rmrr;
 263         struct acpi_dmar_atsr *atsr;
 264         struct acpi_dmar_rhsa *rhsa;
 265
 266         switch (header->type) {
 267         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 268                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 269                                     header);
 270                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 271                         (unsigned long long)drhd->address, drhd->flags);
 272                 break;
 273         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 274                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 275                                     header);
 276                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 277                         (unsigned long long)rmrr->base_address,
 278                         (unsigned long long)rmrr->end_address);
 279                 break;
 280         case ACPI_DMAR_TYPE_ATSR:
 281                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 282                 pr_info("ATSR flags: %#x\n", atsr->flags);
 283                 break;
 284         case ACPI_DMAR_HARDWARE_AFFINITY:
 285                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 286                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 287                        (unsigned long long)rhsa->base_address,
 288                        rhsa->proximity_domain);
 289                 break;
 290         }
 291 }
 292
 293 /**
 294  * dmar_table_detect - checks to see if the platform supports DMAR devices
 295  */
 296 static int __init dmar_table_detect(void)
 297 {
 298         acpi_status status = AE_OK;
 299
 300         /* if we could find DMAR table, then there are DMAR devices */
 301         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 302                                 (struct acpi_table_header **)&dmar_tbl,
 303                                 &dmar_tbl_size);
 304
 305         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 306                 pr_warn("Unable to map DMAR\n");
 307                 status = AE_NOT_FOUND;
 308         }
 309
 310         return (ACPI_SUCCESS(status) ? 1 : 0);
 311 }
 312
 313 /**
 314  * parse_dmar_table - parses the DMA reporting table
 315  */
 316 static int __init
 317 parse_dmar_table(void)
 318 {
 319         struct acpi_table_dmar *dmar;
 320         struct acpi_dmar_header *entry_header;
 321         int ret = 0;
 322         int drhd_count = 0;
 323
 324         /*
 325          * Do it again, earlier dmar_tbl mapping could be mapped with
 326          * fixed map.
 327          */
 328         dmar_table_detect();
 329
 330         /*
 331          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 332          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 333          */
 334         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 335
 336         dmar = (struct acpi_table_dmar *)dmar_tbl;
 337         if (!dmar)
 338                 return -ENODEV;
 339
 340         if (dmar->width < PAGE_SHIFT - 1) {
 341                 pr_warn("Invalid DMAR haw\n");
 342                 return -EINVAL;
 343         }
 344
 345         pr_info("Host address width %d\n", dmar->width + 1);
 346
 347         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 348         while (((unsigned long)entry_header) <
 349                         (((unsigned long)dmar) + dmar_tbl->length)) {
 350                 /* Avoid looping forever on bad ACPI tables */
 351                 if (entry_header->length == 0) {
 352                         pr_warn("Invalid 0-length structure\n");
 353                         ret = -EINVAL;
 354                         break;
 355                 }
 356
 357                 dmar_table_print_dmar_entry(entry_header);
 358
 359                 switch (entry_header->type) {
 360                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 361                         drhd_count++;
 362                         ret = dmar_parse_one_drhd(entry_header);
 363                         break;
 364                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 365                         ret = dmar_parse_one_rmrr(entry_header);
 366                         break;
 367                 case ACPI_DMAR_TYPE_ATSR:
 368                         ret = dmar_parse_one_atsr(entry_header);
 369                         break;
 370                 case ACPI_DMAR_HARDWARE_AFFINITY:
 371 #ifdef CONFIG_ACPI_NUMA
 372                         ret = dmar_parse_one_rhsa(entry_header);
 373 #endif
 374                         break;
 375                 default:
 376                         pr_warn("Unknown DMAR structure type %d\n",
 377                                 entry_header->type);
 378                         ret = 0; /* for forward compatibility */
 379                         break;
 380                 }
 381                 if (ret)
 382                         break;
 383
 384                 entry_header = ((void *)entry_header + entry_header->length);
 385         }
 386         if (drhd_count == 0)
 387                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 388         return ret;
 389 }
 390
 391 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 392                           struct pci_dev *dev)
 393 {
 394         int index;
 395
 396         while (dev) {
 397                 for (index = 0; index < cnt; index++)
 398                         if (dev == devices[index])
 399                                 return 1;
 400
 401                 /* Check our parent */
 402                 dev = dev->bus->self;
 403         }
 404
 405         return 0;
 406 }
 407
 408 struct dmar_drhd_unit *
 409 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 410 {
 411         struct dmar_drhd_unit *dmaru = NULL;
 412         struct acpi_dmar_hardware_unit *drhd;
 413
 414         dev = pci_physfn(dev);
 415
 416         for_each_drhd_unit(dmaru) {
 417                 drhd = container_of(dmaru->hdr,
 418                                     struct acpi_dmar_hardware_unit,
 419                                     header);
 420
 421                 if (dmaru->include_all &&
 422                     drhd->segment == pci_domain_nr(dev->bus))
 423                         return dmaru;
 424
 425                 if (dmar_pci_device_match(dmaru->devices,
 426                                           dmaru->devices_cnt, dev))
 427                         return dmaru;
 428         }
 429
 430         return NULL;
 431 }
 432
 433 int __init dmar_dev_scope_init(void)
 434 {
 435         static int dmar_dev_scope_initialized;
 436         struct dmar_drhd_unit *drhd, *drhd_n;
 437         int ret = -ENODEV;
 438
 439         if (dmar_dev_scope_initialized)
 440                 return dmar_dev_scope_initialized;
 441
 442         if (list_empty(&dmar_drhd_units))
 443                 goto fail;
 444
 445         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 446                 ret = dmar_parse_dev(drhd);
 447                 if (ret)
 448                         goto fail;
 449         }
 450
 451         ret = dmar_parse_rmrr_atsr_dev();
 452         if (ret)
 453                 goto fail;
 454
 455         dmar_dev_scope_initialized = 1;
 456         return 0;
 457
 458 fail:
 459         dmar_dev_scope_initialized = ret;
 460         return ret;
 461 }
 462
 463
 464 int __init dmar_table_init(void)
 465 {
 466         static int dmar_table_initialized;
 467         int ret;
 468
 469         if (dmar_table_initialized)
 470                 return 0;
 471
 472         dmar_table_initialized = 1;
 473
 474         ret = parse_dmar_table();
 475         if (ret) {
 476                 if (ret != -ENODEV)
 477                         pr_info("parse DMAR table failure.\n");
 478                 return ret;
 479         }
 480
 481         if (list_empty(&dmar_drhd_units)) {
 482                 pr_info("No DMAR devices found\n");
 483                 return -ENODEV;
 484         }
 485
 486         return 0;
 487 }
 488
 489 static void warn_invalid_dmar(u64 addr, const char *message)
 490 {
 491         WARN_TAINT_ONCE(
 492                 1, TAINT_FIRMWARE_WORKAROUND,
 493                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 494                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 495                 addr, message,
 496                 dmi_get_system_info(DMI_BIOS_VENDOR),
 497                 dmi_get_system_info(DMI_BIOS_VERSION),
 498                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 499 }
 500
 501 static int __init check_zero_address(void)
 502 {
 503         struct acpi_table_dmar *dmar;
 504         struct acpi_dmar_header *entry_header;
 505         struct acpi_dmar_hardware_unit *drhd;
 506
 507         dmar = (struct acpi_table_dmar *)dmar_tbl;
 508         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 509
 510         while (((unsigned long)entry_header) <
 511                         (((unsigned long)dmar) + dmar_tbl->length)) {
 512                 /* Avoid looping forever on bad ACPI tables */
 513                 if (entry_header->length == 0) {
 514                         pr_warn("Invalid 0-length structure\n");
 515                         return 0;
 516                 }
 517
 518                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 519                         void __iomem *addr;
 520                         u64 cap, ecap;
 521
 522                         drhd = (void *)entry_header;
 523                         if (!drhd->address) {
 524                                 warn_invalid_dmar(0, "");
 525                                 goto failed;
 526                         }
 527
 528                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 529                         if (!addr ) {
 530                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 531                                 goto failed;
 532                         }
 533                         cap = dmar_readq(addr + DMAR_CAP_REG);
 534                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 535                         early_iounmap(addr, VTD_PAGE_SIZE);
 536                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 537                                 warn_invalid_dmar(drhd->address,
 538                                                   " returns all ones");
 539                                 goto failed;
 540                         }
 541                 }
 542
 543                 entry_header = ((void *)entry_header + entry_header->length);
 544         }
 545         return 1;
 546
 547 failed:
 548         return 0;
 549 }
 550
 551 int __init detect_intel_iommu(void)
 552 {
 553         int ret;
 554
 555         ret = dmar_table_detect();
 556         if (ret)
 557                 ret = check_zero_address();
 558         {
 559                 struct acpi_table_dmar *dmar;
 560
 561                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 562
 563                 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
 564                     dmar->flags & 0x1)
 565                         pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
 566
 567                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 568                         iommu_detected = 1;
 569                         /* Make sure ACS will be enabled */
 570                         pci_request_acs();
 571                 }
 572
 573 #ifdef CONFIG_X86
 574                 if (ret)
 575                         x86_init.iommu.iommu_init = intel_iommu_init;
 576 #endif
 577         }
 578         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 579         dmar_tbl = NULL;
 580
 581         return ret ? 1 : -ENODEV;
 582 }
 583
 584
 585 static void unmap_iommu(struct intel_iommu *iommu)
 586 {
 587         iounmap(iommu->reg);
 588         release_mem_region(iommu->reg_phys, iommu->reg_size);
 589 }
 590
 591 /**
 592  * map_iommu: map the iommu's registers
 593  * @iommu: the iommu to map
 594  * @phys_addr: the physical address of the base resgister
 595  *
 596  * Memory map the iommu's registers.  Start w/ a single page, and
 597  * possibly expand if that turns out to be insufficent.
 598  */
 599 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 600 {
 601         int map_size, err=0;
 602
 603         iommu->reg_phys = phys_addr;
 604         iommu->reg_size = VTD_PAGE_SIZE;
 605
 606         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 607                 pr_err("IOMMU: can't reserve memory\n");
 608                 err = -EBUSY;
 609                 goto out;
 610         }
 611
 612         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 613         if (!iommu->reg) {
 614                 pr_err("IOMMU: can't map the region\n");
 615                 err = -ENOMEM;
 616                 goto release;
 617         }
 618
 619         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 620         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 621
 622         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 623                 err = -EINVAL;
 624                 warn_invalid_dmar(phys_addr, " returns all ones");
 625                 goto unmap;
 626         }
 627
 628         /* the registers might be more than one page */
 629         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 630                          cap_max_fault_reg_offset(iommu->cap));
 631         map_size = VTD_PAGE_ALIGN(map_size);
 632         if (map_size > iommu->reg_size) {
 633                 iounmap(iommu->reg);
 634                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 635                 iommu->reg_size = map_size;
 636                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 637                                         iommu->name)) {
 638                         pr_err("IOMMU: can't reserve memory\n");
 639                         err = -EBUSY;
 640                         goto out;
 641                 }
 642                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 643                 if (!iommu->reg) {
 644                         pr_err("IOMMU: can't map the region\n");
 645                         err = -ENOMEM;
 646                         goto release;
 647                 }
 648         }
 649         err = 0;
 650         goto out;
 651
 652 unmap:
 653         iounmap(iommu->reg);
 654 release:
 655         release_mem_region(iommu->reg_phys, iommu->reg_size);
 656 out:
 657         return err;
 658 }
 659
 660 int alloc_iommu(struct dmar_drhd_unit *drhd)
 661 {
 662         struct intel_iommu *iommu;
 663         u32 ver, sts;
 664         static int iommu_allocated = 0;
 665         int agaw = 0;
 666         int msagaw = 0;
 667         int err;
 668
 669         if (!drhd->reg_base_addr) {
 670                 warn_invalid_dmar(0, "");
 671                 return -EINVAL;
 672         }
 673
 674         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 675         if (!iommu)
 676                 return -ENOMEM;
 677
 678         iommu->seq_id = iommu_allocated++;
 679         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 680
 681         err = map_iommu(iommu, drhd->reg_base_addr);
 682         if (err) {
 683                 pr_err("IOMMU: failed to map %s\n", iommu->name);
 684                 goto error;
 685         }
 686
 687         err = -EINVAL;
 688         agaw = iommu_calculate_agaw(iommu);
 689         if (agaw < 0) {
 690                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 691                         iommu->seq_id);
 692                 goto err_unmap;
 693         }
 694         msagaw = iommu_calculate_max_sagaw(iommu);
 695         if (msagaw < 0) {
 696                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 697                         iommu->seq_id);
 698                 goto err_unmap;
 699         }
 700         iommu->agaw = agaw;
 701         iommu->msagaw = msagaw;
 702
 703         iommu->node = -1;
 704
 705         ver = readl(iommu->reg + DMAR_VER_REG);
 706         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 707                 iommu->seq_id,
 708                 (unsigned long long)drhd->reg_base_addr,
 709                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 710                 (unsigned long long)iommu->cap,
 711                 (unsigned long long)iommu->ecap);
 712
 713         /* Reflect status in gcmd */
 714         sts = readl(iommu->reg + DMAR_GSTS_REG);
 715         if (sts & DMA_GSTS_IRES)
 716                 iommu->gcmd |= DMA_GCMD_IRE;
 717         if (sts & DMA_GSTS_TES)
 718                 iommu->gcmd |= DMA_GCMD_TE;
 719         if (sts & DMA_GSTS_QIES)
 720                 iommu->gcmd |= DMA_GCMD_QIE;
 721
 722         raw_spin_lock_init(&iommu->register_lock);
 723
 724         drhd->iommu = iommu;
 725         return 0;
 726
 727  err_unmap:
 728         unmap_iommu(iommu);
 729  error:
 730         kfree(iommu);
 731         return err;
 732 }
 733
 734 void free_iommu(struct intel_iommu *iommu)
 735 {
 736         if (!iommu)
 737                 return;
 738
 739         free_dmar_iommu(iommu);
 740
 741         if (iommu->reg)
 742                 unmap_iommu(iommu);
 743
 744         kfree(iommu);
 745 }
 746
 747 /*
 748  * Reclaim all the submitted descriptors which have completed its work.
 749  */
 750 static inline void reclaim_free_desc(struct q_inval *qi)
 751 {
 752         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 753                qi->desc_status[qi->free_tail] == QI_ABORT) {
 754                 qi->desc_status[qi->free_tail] = QI_FREE;
 755                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 756                 qi->free_cnt++;
 757         }
 758 }
 759
 760 static int qi_check_fault(struct intel_iommu *iommu, int index)
 761 {
 762         u32 fault;
 763         int head, tail;
 764         struct q_inval *qi = iommu->qi;
 765         int wait_index = (index + 1) % QI_LENGTH;
 766
 767         if (qi->desc_status[wait_index] == QI_ABORT)
 768                 return -EAGAIN;
 769
 770         fault = readl(iommu->reg + DMAR_FSTS_REG);
 771
 772         /*
 773          * If IQE happens, the head points to the descriptor associated
 774          * with the error. No new descriptors are fetched until the IQE
 775          * is cleared.
 776          */
 777         if (fault & DMA_FSTS_IQE) {
 778                 head = readl(iommu->reg + DMAR_IQH_REG);
 779                 if ((head >> DMAR_IQ_SHIFT) == index) {
 780                         pr_err("VT-d detected invalid descriptor: "
 781                                 "low=%llx, high=%llx\n",
 782                                 (unsigned long long)qi->desc[index].low,
 783                                 (unsigned long long)qi->desc[index].high);
 784                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 785                                         sizeof(struct qi_desc));
 786                         __iommu_flush_cache(iommu, &qi->desc[index],
 787                                         sizeof(struct qi_desc));
 788                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 789                         return -EINVAL;
 790                 }
 791         }
 792
 793         /*
 794          * If ITE happens, all pending wait_desc commands are aborted.
 795          * No new descriptors are fetched until the ITE is cleared.
 796          */
 797         if (fault & DMA_FSTS_ITE) {
 798                 head = readl(iommu->reg + DMAR_IQH_REG);
 799                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 800                 head |= 1;
 801                 tail = readl(iommu->reg + DMAR_IQT_REG);
 802                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 803
 804                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 805
 806                 do {
 807                         if (qi->desc_status[head] == QI_IN_USE)
 808                                 qi->desc_status[head] = QI_ABORT;
 809                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 810                 } while (head != tail);
 811
 812                 if (qi->desc_status[wait_index] == QI_ABORT)
 813                         return -EAGAIN;
 814         }
 815
 816         if (fault & DMA_FSTS_ICE)
 817                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 818
 819         return 0;
 820 }
 821
 822 /*
 823  * Submit the queued invalidation descriptor to the remapping
 824  * hardware unit and wait for its completion.
 825  */
 826 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 827 {
 828         int rc;
 829         struct q_inval *qi = iommu->qi;
 830         struct qi_desc *hw, wait_desc;
 831         int wait_index, index;
 832         unsigned long flags;
 833
 834         if (!qi)
 835                 return 0;
 836
 837         hw = qi->desc;
 838
 839 restart:
 840         rc = 0;
 841
 842         raw_spin_lock_irqsave(&qi->q_lock, flags);
 843         while (qi->free_cnt < 3) {
 844                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 845                 cpu_relax();
 846                 raw_spin_lock_irqsave(&qi->q_lock, flags);
 847         }
 848
 849         index = qi->free_head;
 850         wait_index = (index + 1) % QI_LENGTH;
 851
 852         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 853
 854         hw[index] = *desc;
 855
 856         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 857                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 858         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 859
 860         hw[wait_index] = wait_desc;
 861
 862         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 863         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 864
 865         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 866         qi->free_cnt -= 2;
 867
 868         /*
 869          * update the HW tail register indicating the presence of
 870          * new descriptors.
 871          */
 872         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 873
 874         while (qi->desc_status[wait_index] != QI_DONE) {
 875                 /*
 876                  * We will leave the interrupts disabled, to prevent interrupt
 877                  * context to queue another cmd while a cmd is already submitted
 878                  * and waiting for completion on this cpu. This is to avoid
 879                  * a deadlock where the interrupt context can wait indefinitely
 880                  * for free slots in the queue.
 881                  */
 882                 rc = qi_check_fault(iommu, index);
 883                 if (rc)
 884                         break;
 885
 886                 raw_spin_unlock(&qi->q_lock);
 887                 cpu_relax();
 888                 raw_spin_lock(&qi->q_lock);
 889         }
 890
 891         qi->desc_status[index] = QI_DONE;
 892
 893         reclaim_free_desc(qi);
 894         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 895
 896         if (rc == -EAGAIN)
 897                 goto restart;
 898
 899         return rc;
 900 }
 901
 902 /*
 903  * Flush the global interrupt entry cache.
 904  */
 905 void qi_global_iec(struct intel_iommu *iommu)
 906 {
 907         struct qi_desc desc;
 908
 909         desc.low = QI_IEC_TYPE;
 910         desc.high = 0;
 911
 912         /* should never fail */
 913         qi_submit_sync(&desc, iommu);
 914 }
 915
 916 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 917                       u64 type)
 918 {
 919         struct qi_desc desc;
 920
 921         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 922                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 923         desc.high = 0;
 924
 925         qi_submit_sync(&desc, iommu);
 926 }
 927
 928 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 929                     unsigned int size_order, u64 type)
 930 {
 931         u8 dw = 0, dr = 0;
 932
 933         struct qi_desc desc;
 934         int ih = 0;
 935
 936         if (cap_write_drain(iommu->cap))
 937                 dw = 1;
 938
 939         if (cap_read_drain(iommu->cap))
 940                 dr = 1;
 941
 942         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 943                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 944         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 945                 | QI_IOTLB_AM(size_order);
 946
 947         qi_submit_sync(&desc, iommu);
 948 }
 949
 950 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 951                         u64 addr, unsigned mask)
 952 {
 953         struct qi_desc desc;
 954
 955         if (mask) {
 956                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 957                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 958                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 959         } else
 960                 desc.high = QI_DEV_IOTLB_ADDR(addr);
 961
 962         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 963                 qdep = 0;
 964
 965         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 966                    QI_DIOTLB_TYPE;
 967
 968         qi_submit_sync(&desc, iommu);
 969 }
 970
 971 /*
 972  * Disable Queued Invalidation interface.
 973  */
 974 void dmar_disable_qi(struct intel_iommu *iommu)
 975 {
 976         unsigned long flags;
 977         u32 sts;
 978         cycles_t start_time = get_cycles();
 979
 980         if (!ecap_qis(iommu->ecap))
 981                 return;
 982
 983         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 984
 985         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 986         if (!(sts & DMA_GSTS_QIES))
 987                 goto end;
 988
 989         /*
 990          * Give a chance to HW to complete the pending invalidation requests.
 991          */
 992         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 993                 readl(iommu->reg + DMAR_IQH_REG)) &&
 994                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 995                 cpu_relax();
 996
 997         iommu->gcmd &= ~DMA_GCMD_QIE;
 998         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 999
1000         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1001                       !(sts & DMA_GSTS_QIES), sts);
1002 end:
1003         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1004 }
1005
1006 /*
1007  * Enable queued invalidation.
1008  */
1009 static void __dmar_enable_qi(struct intel_iommu *iommu)
1010 {
1011         u32 sts;
1012         unsigned long flags;
1013         struct q_inval *qi = iommu->qi;
1014
1015         qi->free_head = qi->free_tail = 0;
1016         qi->free_cnt = QI_LENGTH;
1017
1018         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1019
1020         /* write zero to the tail reg */
1021         writel(0, iommu->reg + DMAR_IQT_REG);
1022
1023         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1024
1025         iommu->gcmd |= DMA_GCMD_QIE;
1026         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1027
1028         /* Make sure hardware complete it */
1029         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1030
1031         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1032 }
1033
1034 /*
1035  * Enable Queued Invalidation interface. This is a must to support
1036  * interrupt-remapping. Also used by DMA-remapping, which replaces
1037  * register based IOTLB invalidation.
1038  */
1039 int dmar_enable_qi(struct intel_iommu *iommu)
1040 {
1041         struct q_inval *qi;
1042         struct page *desc_page;
1043
1044         if (!ecap_qis(iommu->ecap))
1045                 return -ENOENT;
1046
1047         /*
1048          * queued invalidation is already setup and enabled.
1049          */
1050         if (iommu->qi)
1051                 return 0;
1052
1053         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1054         if (!iommu->qi)
1055                 return -ENOMEM;
1056
1057         qi = iommu->qi;
1058
1059
1060         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1061         if (!desc_page) {
1062                 kfree(qi);
1063                 iommu->qi = 0;
1064                 return -ENOMEM;
1065         }
1066
1067         qi->desc = page_address(desc_page);
1068
1069         qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1070         if (!qi->desc_status) {
1071                 free_page((unsigned long) qi->desc);
1072                 kfree(qi);
1073                 iommu->qi = 0;
1074                 return -ENOMEM;
1075         }
1076
1077         qi->free_head = qi->free_tail = 0;
1078         qi->free_cnt = QI_LENGTH;
1079
1080         raw_spin_lock_init(&qi->q_lock);
1081
1082         __dmar_enable_qi(iommu);
1083
1084         return 0;
1085 }
1086
1087 /* iommu interrupt handling. Most stuff are MSI-like. */
1088
1089 enum faulttype {
1090         DMA_REMAP,
1091         INTR_REMAP,
1092         UNKNOWN,
1093 };
1094
1095 static const char *dma_remap_fault_reasons[] =
1096 {
1097         "Software",
1098         "Present bit in root entry is clear",
1099         "Present bit in context entry is clear",
1100         "Invalid context entry",
1101         "Access beyond MGAW",
1102         "PTE Write access is not set",
1103         "PTE Read access is not set",
1104         "Next page table ptr is invalid",
1105         "Root table address invalid",
1106         "Context table ptr is invalid",
1107         "non-zero reserved fields in RTP",
1108         "non-zero reserved fields in CTP",
1109         "non-zero reserved fields in PTE",
1110         "PCE for translation request specifies blocking",
1111 };
1112
1113 static const char *irq_remap_fault_reasons[] =
1114 {
1115         "Detected reserved fields in the decoded interrupt-remapped request",
1116         "Interrupt index exceeded the interrupt-remapping table size",
1117         "Present field in the IRTE entry is clear",
1118         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1119         "Detected reserved fields in the IRTE entry",
1120         "Blocked a compatibility format interrupt request",
1121         "Blocked an interrupt request due to source-id verification failure",
1122 };
1123
1124 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1125
1126 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1127 {
1128         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1129                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1130                 *fault_type = INTR_REMAP;
1131                 return irq_remap_fault_reasons[fault_reason - 0x20];
1132         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1133                 *fault_type = DMA_REMAP;
1134                 return dma_remap_fault_reasons[fault_reason];
1135         } else {
1136                 *fault_type = UNKNOWN;
1137                 return "Unknown";
1138         }
1139 }
1140
1141 void dmar_msi_unmask(struct irq_data *data)
1142 {
1143         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1144         unsigned long flag;
1145
1146         /* unmask it */
1147         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1148         writel(0, iommu->reg + DMAR_FECTL_REG);
1149         /* Read a reg to force flush the post write */
1150         readl(iommu->reg + DMAR_FECTL_REG);
1151         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1152 }
1153
1154 void dmar_msi_mask(struct irq_data *data)
1155 {
1156         unsigned long flag;
1157         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1158
1159         /* mask it */
1160         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1161         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1162         /* Read a reg to force flush the post write */
1163         readl(iommu->reg + DMAR_FECTL_REG);
1164         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1165 }
1166
1167 void dmar_msi_write(int irq, struct msi_msg *msg)
1168 {
1169         struct intel_iommu *iommu = irq_get_handler_data(irq);
1170         unsigned long flag;
1171
1172         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1173         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1174         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1175         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1176         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1177 }
1178
1179 void dmar_msi_read(int irq, struct msi_msg *msg)
1180 {
1181         struct intel_iommu *iommu = irq_get_handler_data(irq);
1182         unsigned long flag;
1183
1184         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1185         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1186         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1187         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1188         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1189 }
1190
1191 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1192                 u8 fault_reason, u16 source_id, unsigned long long addr)
1193 {
1194         const char *reason;
1195         int fault_type;
1196
1197         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1198
1199         if (fault_type == INTR_REMAP)
1200                 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1201                        "fault index %llx\n"
1202                         "INTR-REMAP:[fault reason %02d] %s\n",
1203                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1204                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1205                         fault_reason, reason);
1206         else
1207                 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1208                        "fault addr %llx \n"
1209                        "DMAR:[fault reason %02d] %s\n",
1210                        (type ? "DMA Read" : "DMA Write"),
1211                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1212                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1213         return 0;
1214 }
1215
1216 #define PRIMARY_FAULT_REG_LEN (16)
1217 irqreturn_t dmar_fault(int irq, void *dev_id)
1218 {
1219         struct intel_iommu *iommu = dev_id;
1220         int reg, fault_index;
1221         u32 fault_status;
1222         unsigned long flag;
1223
1224         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1225         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1226         if (fault_status)
1227                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1228
1229         /* TBD: ignore advanced fault log currently */
1230         if (!(fault_status & DMA_FSTS_PPF))
1231                 goto unlock_exit;
1232
1233         fault_index = dma_fsts_fault_record_index(fault_status);
1234         reg = cap_fault_reg_offset(iommu->cap);
1235         while (1) {
1236                 u8 fault_reason;
1237                 u16 source_id;
1238                 u64 guest_addr;
1239                 int type;
1240                 u32 data;
1241
1242                 /* highest 32 bits */
1243                 data = readl(iommu->reg + reg +
1244                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1245                 if (!(data & DMA_FRCD_F))
1246                         break;
1247
1248                 fault_reason = dma_frcd_fault_reason(data);
1249                 type = dma_frcd_type(data);
1250
1251                 data = readl(iommu->reg + reg +
1252                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1253                 source_id = dma_frcd_source_id(data);
1254
1255                 guest_addr = dmar_readq(iommu->reg + reg +
1256                                 fault_index * PRIMARY_FAULT_REG_LEN);
1257                 guest_addr = dma_frcd_page_addr(guest_addr);
1258                 /* clear the fault */
1259                 writel(DMA_FRCD_F, iommu->reg + reg +
1260                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1261
1262                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1263
1264                 dmar_fault_do_one(iommu, type, fault_reason,
1265                                 source_id, guest_addr);
1266
1267                 fault_index++;
1268                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1269                         fault_index = 0;
1270                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1271         }
1272
1273         writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1274
1275 unlock_exit:
1276         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1277         return IRQ_HANDLED;
1278 }
1279
1280 int dmar_set_interrupt(struct intel_iommu *iommu)
1281 {
1282         int irq, ret;
1283
1284         /*
1285          * Check if the fault interrupt is already initialized.
1286          */
1287         if (iommu->irq)
1288                 return 0;
1289
1290         irq = create_irq();
1291         if (!irq) {
1292                 pr_err("IOMMU: no free vectors\n");
1293                 return -EINVAL;
1294         }
1295
1296         irq_set_handler_data(irq, iommu);
1297         iommu->irq = irq;
1298
1299         ret = arch_setup_dmar_msi(irq);
1300         if (ret) {
1301                 irq_set_handler_data(irq, NULL);
1302                 iommu->irq = 0;
1303                 destroy_irq(irq);
1304                 return ret;
1305         }
1306
1307         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1308         if (ret)
1309                 pr_err("IOMMU: can't request irq\n");
1310         return ret;
1311 }
1312
1313 int __init enable_drhd_fault_handling(void)
1314 {
1315         struct dmar_drhd_unit *drhd;
1316
1317         /*
1318          * Enable fault control interrupt.
1319          */
1320         for_each_drhd_unit(drhd) {
1321                 int ret;
1322                 struct intel_iommu *iommu = drhd->iommu;
1323                 u32 fault_status;
1324                 ret = dmar_set_interrupt(iommu);
1325
1326                 if (ret) {
1327                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1328                                (unsigned long long)drhd->reg_base_addr, ret);
1329                         return -1;
1330                 }
1331
1332                 /*
1333                  * Clear any previous faults.
1334                  */
1335                 dmar_fault(iommu->irq, iommu);
1336                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1337                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1338         }
1339
1340         return 0;
1341 }
1342
1343 /*
1344  * Re-enable Queued Invalidation interface.
1345  */
1346 int dmar_reenable_qi(struct intel_iommu *iommu)
1347 {
1348         if (!ecap_qis(iommu->ecap))
1349                 return -ENOENT;
1350
1351         if (!iommu->qi)
1352                 return -ENOENT;
1353
1354         /*
1355          * First disable queued invalidation.
1356          */
1357         dmar_disable_qi(iommu);
1358         /*
1359          * Then enable queued invalidation again. Since there is no pending
1360          * invalidation requests now, it's safe to re-enable queued
1361          * invalidation.
1362          */
1363         __dmar_enable_qi(iommu);
1364
1365         return 0;
1366 }
1367
1368 /*
1369  * Check interrupt remapping support in DMAR table description.
1370  */
1371 int __init dmar_ir_support(void)
1372 {
1373         struct acpi_table_dmar *dmar;
1374         dmar = (struct acpi_table_dmar *)dmar_tbl;
1375         if (!dmar)
1376                 return 0;
1377         return dmar->flags & 0x1;
1378 }
1379 IOMMU_INIT_POST(detect_intel_iommu);