arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/gfp.h>
   6 #include <linux/pci.h>
   7 #include <linux/kmemleak.h>
   8
   9 #include <asm/proto.h>
  10 #include <asm/dma.h>
  11 #include <asm/iommu.h>
  12 #include <asm/gart.h>
  13 #include <asm/calgary.h>
  14 #include <asm/x86_init.h>
  15 #include <asm/iommu_table.h>
  16
  17 static int forbid_dac __read_mostly;
  18
  19 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  20 EXPORT_SYMBOL(dma_ops);
  21
  22 static int iommu_sac_force __read_mostly;
  23
  24 #ifdef CONFIG_IOMMU_DEBUG
  25 int panic_on_overflow __read_mostly = 1;
  26 int force_iommu __read_mostly = 1;
  27 #else
  28 int panic_on_overflow __read_mostly = 0;
  29 int force_iommu __read_mostly = 0;
  30 #endif
  31
  32 int iommu_merge __read_mostly = 0;
  33
  34 int no_iommu __read_mostly;
  35 /* Set this to 1 if there is a HW IOMMU in the system */
  36 int iommu_detected __read_mostly = 0;
  37
  38 /*
  39  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  40  * If this variable is 1, IOMMU implementations do no DMA translation for
  41  * devices and allow every device to access to whole physical memory. This is
  42  * useful if a user wants to use an IOMMU only for KVM device assignment to
  43  * guests and not for driver dma translation.
  44  */
  45 int iommu_pass_through __read_mostly;
  46
  47 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
  48
  49 /* Dummy device used for NULL arguments (normally ISA). */
  50 struct device x86_dma_fallback_dev = {
  51         .init_name = "fallback device",
  52         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  53         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  54 };
  55 EXPORT_SYMBOL(x86_dma_fallback_dev);
  56
  57 /* Number of entries preallocated for DMA-API debugging */
  58 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  59
  60 int dma_set_mask(struct device *dev, u64 mask)
  61 {
  62         if (!dev->dma_mask || !dma_supported(dev, mask))
  63                 return -EIO;
  64
  65         *dev->dma_mask = mask;
  66
  67         return 0;
  68 }
  69 EXPORT_SYMBOL(dma_set_mask);
  70
  71 void __init pci_iommu_alloc(void)
  72 {
  73         struct iommu_table_entry *p;
  74
  75         sort_iommu_table(__iommu_table, __iommu_table_end);
  76         check_iommu_entries(__iommu_table, __iommu_table_end);
  77
  78         for (p = __iommu_table; p < __iommu_table_end; p++) {
  79                 if (p && p->detect && p->detect() > 0) {
  80                         p->flags |= IOMMU_DETECTED;
  81                         if (p->early_init)
  82                                 p->early_init();
  83                         if (p->flags & IOMMU_FINISH_IF_DETECTED)
  84                                 break;
  85                 }
  86         }
  87 }
  88 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
  89                                  dma_addr_t *dma_addr, gfp_t flag)
  90 {
  91         unsigned long dma_mask;
  92         struct page *page;
  93         dma_addr_t addr;
  94
  95         dma_mask = dma_alloc_coherent_mask(dev, flag);
  96
  97         flag |= __GFP_ZERO;
  98 again:
  99         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 100         if (!page)
 101                 return NULL;
 102
 103         addr = page_to_phys(page);
 104         if (addr + size > dma_mask) {
 105                 __free_pages(page, get_order(size));
 106
 107                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 108                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 109                         goto again;
 110                 }
 111
 112                 return NULL;
 113         }
 114
 115         *dma_addr = addr;
 116         return page_address(page);
 117 }
 118
 119 /*
 120  * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
 121  * parameter documentation.
 122  */
 123 static __init int iommu_setup(char *p)
 124 {
 125         iommu_merge = 1;
 126
 127         if (!p)
 128                 return -EINVAL;
 129
 130         while (*p) {
 131                 if (!strncmp(p, "off", 3))
 132                         no_iommu = 1;
 133                 /* gart_parse_options has more force support */
 134                 if (!strncmp(p, "force", 5))
 135                         force_iommu = 1;
 136                 if (!strncmp(p, "noforce", 7)) {
 137                         iommu_merge = 0;
 138                         force_iommu = 0;
 139                 }
 140
 141                 if (!strncmp(p, "biomerge", 8)) {
 142                         iommu_merge = 1;
 143                         force_iommu = 1;
 144                 }
 145                 if (!strncmp(p, "panic", 5))
 146                         panic_on_overflow = 1;
 147                 if (!strncmp(p, "nopanic", 7))
 148                         panic_on_overflow = 0;
 149                 if (!strncmp(p, "merge", 5)) {
 150                         iommu_merge = 1;
 151                         force_iommu = 1;
 152                 }
 153                 if (!strncmp(p, "nomerge", 7))
 154                         iommu_merge = 0;
 155                 if (!strncmp(p, "forcesac", 8))
 156                         iommu_sac_force = 1;
 157                 if (!strncmp(p, "allowdac", 8))
 158                         forbid_dac = 0;
 159                 if (!strncmp(p, "nodac", 5))
 160                         forbid_dac = 1;
 161                 if (!strncmp(p, "usedac", 6)) {
 162                         forbid_dac = -1;
 163                         return 1;
 164                 }
 165 #ifdef CONFIG_SWIOTLB
 166                 if (!strncmp(p, "soft", 4))
 167                         swiotlb = 1;
 168 #endif
 169                 if (!strncmp(p, "pt", 2))
 170                         iommu_pass_through = 1;
 171
 172                 gart_parse_options(p);
 173
 174 #ifdef CONFIG_CALGARY_IOMMU
 175                 if (!strncmp(p, "calgary", 7))
 176                         use_calgary = 1;
 177 #endif /* CONFIG_CALGARY_IOMMU */
 178
 179                 p += strcspn(p, ",");
 180                 if (*p == ',')
 181                         ++p;
 182         }
 183         return 0;
 184 }
 185 early_param("iommu", iommu_setup);
 186
 187 int dma_supported(struct device *dev, u64 mask)
 188 {
 189         struct dma_map_ops *ops = get_dma_ops(dev);
 190
 191 #ifdef CONFIG_PCI
 192         if (mask > 0xffffffff && forbid_dac > 0) {
 193                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 194                 return 0;
 195         }
 196 #endif
 197
 198         if (ops->dma_supported)
 199                 return ops->dma_supported(dev, mask);
 200
 201         /* Copied from i386. Doesn't make much sense, because it will
 202            only work for pci_alloc_coherent.
 203            The caller just has to use GFP_DMA in this case. */
 204         if (mask < DMA_BIT_MASK(24))
 205                 return 0;
 206
 207         /* Tell the device to use SAC when IOMMU force is on.  This
 208            allows the driver to use cheaper accesses in some cases.
 209
 210            Problem with this is that if we overflow the IOMMU area and
 211            return DAC as fallback address the device may not handle it
 212            correctly.
 213
 214            As a special case some controllers have a 39bit address
 215            mode that is as efficient as 32bit (aic79xx). Don't force
 216            SAC for these.  Assume all masks <= 40 bits are of this
 217            type. Normally this doesn't make any difference, but gives
 218            more gentle handling of IOMMU overflow. */
 219         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 220                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 221                 return 0;
 222         }
 223
 224         return 1;
 225 }
 226 EXPORT_SYMBOL(dma_supported);
 227
 228 static int __init pci_iommu_init(void)
 229 {
 230         struct iommu_table_entry *p;
 231         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 232
 233 #ifdef CONFIG_PCI
 234         dma_debug_add_bus(&pci_bus_type);
 235 #endif
 236         x86_init.iommu.iommu_init();
 237
 238         for (p = __iommu_table; p < __iommu_table_end; p++) {
 239                 if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
 240                         p->late_init();
 241         }
 242
 243         return 0;
 244 }
 245 /* Must execute after PCI subsystem */
 246 rootfs_initcall(pci_iommu_init);
 247
 248 #ifdef CONFIG_PCI
 249 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 250
 251 static __devinit void via_no_dac(struct pci_dev *dev)
 252 {
 253         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 254                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 255                 forbid_dac = 1;
 256         }
 257 }
 258 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 259 #endif