Commit | Line | Data |
---|---|---|
8ab89567 SN |
1 | /* |
2 | * Intel I/OAT DMA Linux driver | |
211a22ce | 3 | * Copyright(c) 2007 - 2009 Intel Corporation. |
8ab89567 SN |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
8ab89567 SN |
14 | * The full GNU General Public License is included in this distribution in |
15 | * the file called "COPYING". | |
16 | * | |
17 | */ | |
18 | ||
19 | /* | |
20 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
21 | * copy operations. | |
22 | */ | |
23 | ||
24 | #include <linux/init.h> | |
25 | #include <linux/module.h> | |
26 | #include <linux/pci.h> | |
27 | #include <linux/interrupt.h> | |
2ed6dc34 | 28 | #include <linux/dca.h> |
5a0e3ad6 | 29 | #include <linux/slab.h> |
b6c52c63 | 30 | #include <linux/acpi.h> |
584ec227 | 31 | #include "dma.h" |
5cbafa65 | 32 | #include "dma_v2.h" |
584ec227 DW |
33 | #include "registers.h" |
34 | #include "hw.h" | |
8ab89567 | 35 | |
5149fd01 | 36 | MODULE_VERSION(IOAT_DMA_VERSION); |
bf40a686 | 37 | MODULE_LICENSE("Dual BSD/GPL"); |
8ab89567 SN |
38 | MODULE_AUTHOR("Intel Corporation"); |
39 | ||
40 | static struct pci_device_id ioat_pci_tbl[] = { | |
7f1b358a | 41 | /* I/OAT v3 platforms */ |
a6417dd5 RD |
42 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, |
43 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, | |
44 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, | |
45 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, | |
46 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, | |
47 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, | |
48 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, | |
49 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, | |
b265b11f TP |
50 | |
51 | /* I/OAT v3.2 platforms */ | |
a6417dd5 RD |
52 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, |
53 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, | |
54 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, | |
55 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, | |
56 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, | |
57 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, | |
58 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, | |
59 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, | |
60 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, | |
61 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, | |
b265b11f | 62 | |
3baef940 DJ |
63 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, |
64 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, | |
65 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, | |
66 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, | |
67 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, | |
68 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, | |
69 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, | |
70 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, | |
71 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, | |
72 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, | |
73 | ||
8eb4da28 DJ |
74 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, |
75 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, | |
76 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, | |
77 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, | |
78 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, | |
79 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, | |
80 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, | |
81 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, | |
82 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, | |
83 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, | |
84 | ||
570727b5 DJ |
85 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, |
86 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, | |
87 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, | |
88 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, | |
89 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, | |
90 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, | |
91 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, | |
92 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, | |
93 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, | |
94 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, | |
95 | ||
0132bcef DJ |
96 | /* I/OAT v3.3 platforms */ |
97 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, | |
98 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, | |
99 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, | |
100 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, | |
101 | ||
68a8cc9e DJ |
102 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) }, |
103 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) }, | |
104 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, | |
105 | { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, | |
106 | ||
8ab89567 SN |
107 | { 0, } |
108 | }; | |
6506cbca | 109 | MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); |
8ab89567 | 110 | |
4bf27b8b GKH |
111 | static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); |
112 | static void ioat_remove(struct pci_dev *pdev); | |
8ab89567 | 113 | |
2ed6dc34 SN |
114 | static int ioat_dca_enabled = 1; |
115 | module_param(ioat_dca_enabled, int, 0644); | |
116 | MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); | |
117 | ||
162b96e6 | 118 | struct kmem_cache *ioat2_cache; |
59056e85 | 119 | struct kmem_cache *ioat3_sed_cache; |
162b96e6 | 120 | |
e6c0b69a DW |
121 | #define DRV_NAME "ioatdma" |
122 | ||
7df7cf06 | 123 | static struct pci_driver ioat_pci_driver = { |
e6c0b69a | 124 | .name = DRV_NAME, |
8ab89567 | 125 | .id_table = ioat_pci_tbl, |
f2427e27 | 126 | .probe = ioat_pci_probe, |
a7d6e3ec | 127 | .remove = ioat_remove, |
8ab89567 SN |
128 | }; |
129 | ||
f2427e27 DW |
130 | static struct ioatdma_device * |
131 | alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) | |
132 | { | |
133 | struct device *dev = &pdev->dev; | |
134 | struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); | |
135 | ||
136 | if (!d) | |
137 | return NULL; | |
138 | d->pdev = pdev; | |
139 | d->reg_base = iobase; | |
140 | return d; | |
141 | } | |
142 | ||
b6c52c63 JL |
143 | /* |
144 | * The dmaengine core assumes that async DMA devices will only be removed | |
145 | * when they not used anymore, or it assumes dma_async_device_unregister() | |
146 | * will only be called by dma driver exit routines. But this assumption is | |
147 | * not true for the IOAT driver, which calls dma_async_device_unregister() | |
148 | * from ioat_remove(). So current IOAT driver doesn't support device | |
149 | * hot-removal because it may cause system crash to hot-remove inuse IOAT | |
150 | * devices. | |
151 | * | |
152 | * This is a hack to disable IOAT devices under ejectable PCI host bridge | |
153 | * so it won't break PCI host bridge hot-removal. | |
154 | */ | |
155 | static bool ioat_pci_has_ejectable_acpi_ancestor(struct pci_dev *pdev) | |
156 | { | |
157 | #ifdef CONFIG_ACPI | |
158 | struct pci_bus *bus = pdev->bus; | |
159 | struct acpi_device *adev; | |
160 | ||
161 | while (bus->parent) | |
162 | bus = bus->parent; | |
163 | for (adev = ACPI_COMPANION(bus->bridge); adev; adev = adev->parent) | |
164 | if (adev->flags.ejectable) | |
165 | return true; | |
166 | #endif | |
167 | ||
168 | return false; | |
169 | } | |
170 | ||
4bf27b8b | 171 | static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
8ab89567 | 172 | { |
e6c0b69a | 173 | void __iomem * const *iomap; |
e6c0b69a | 174 | struct device *dev = &pdev->dev; |
f2427e27 | 175 | struct ioatdma_device *device; |
8ab89567 SN |
176 | int err; |
177 | ||
b6c52c63 JL |
178 | if (ioat_pci_has_ejectable_acpi_ancestor(pdev)) { |
179 | dev_dbg(&pdev->dev, "ignore ejectable IOAT device.\n"); | |
180 | return -ENODEV; | |
181 | } | |
182 | ||
e6c0b69a | 183 | err = pcim_enable_device(pdev); |
8ab89567 | 184 | if (err) |
e6c0b69a | 185 | return err; |
8ab89567 | 186 | |
e6c0b69a | 187 | err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME); |
8ab89567 | 188 | if (err) |
e6c0b69a DW |
189 | return err; |
190 | iomap = pcim_iomap_table(pdev); | |
191 | if (!iomap) | |
192 | return -ENOMEM; | |
8ab89567 | 193 | |
6a35528a | 194 | err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); |
8ab89567 | 195 | if (err) |
284901a9 | 196 | err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); |
8ab89567 | 197 | if (err) |
e6c0b69a | 198 | return err; |
8ab89567 | 199 | |
6a35528a | 200 | err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); |
8ab89567 | 201 | if (err) |
284901a9 | 202 | err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); |
8ab89567 | 203 | if (err) |
e6c0b69a DW |
204 | return err; |
205 | ||
f2427e27 DW |
206 | device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); |
207 | if (!device) | |
208 | return -ENOMEM; | |
c86e1401 | 209 | pci_set_master(pdev); |
f2427e27 DW |
210 | pci_set_drvdata(pdev, device); |
211 | ||
212 | device->version = readb(device->reg_base + IOAT_VER_OFFSET); | |
7f832645 | 213 | if (device->version >= IOAT_VER_3_0) |
f2427e27 DW |
214 | err = ioat3_dma_probe(device, ioat_dca_enabled); |
215 | else | |
e6c0b69a | 216 | return -ENODEV; |
4fac7fa5 | 217 | |
f2427e27 | 218 | if (err) { |
e6c0b69a DW |
219 | dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); |
220 | return -ENODEV; | |
221 | } | |
8ab89567 SN |
222 | |
223 | return 0; | |
8ab89567 SN |
224 | } |
225 | ||
4bf27b8b | 226 | static void ioat_remove(struct pci_dev *pdev) |
8ab89567 | 227 | { |
f2427e27 DW |
228 | struct ioatdma_device *device = pci_get_drvdata(pdev); |
229 | ||
230 | if (!device) | |
231 | return; | |
8ab89567 | 232 | |
4fac7fa5 DW |
233 | dev_err(&pdev->dev, "Removing dma and dca services\n"); |
234 | if (device->dca) { | |
1a5aeeec | 235 | unregister_dca_provider(device->dca, &pdev->dev); |
4fac7fa5 DW |
236 | free_dca_provider(device->dca); |
237 | device->dca = NULL; | |
238 | } | |
f2427e27 | 239 | ioat_dma_remove(device); |
8ab89567 | 240 | } |
8ab89567 SN |
241 | |
242 | static int __init ioat_init_module(void) | |
243 | { | |
59056e85 | 244 | int err = -ENOMEM; |
162b96e6 | 245 | |
5669e31c DW |
246 | pr_info("%s: Intel(R) QuickData Technology Driver %s\n", |
247 | DRV_NAME, IOAT_DMA_VERSION); | |
248 | ||
162b96e6 DW |
249 | ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), |
250 | 0, SLAB_HWCACHE_ALIGN, NULL); | |
251 | if (!ioat2_cache) | |
252 | return -ENOMEM; | |
253 | ||
59056e85 DW |
254 | ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); |
255 | if (!ioat3_sed_cache) | |
256 | goto err_ioat2_cache; | |
257 | ||
162b96e6 DW |
258 | err = pci_register_driver(&ioat_pci_driver); |
259 | if (err) | |
59056e85 DW |
260 | goto err_ioat3_cache; |
261 | ||
262 | return 0; | |
263 | ||
264 | err_ioat3_cache: | |
265 | kmem_cache_destroy(ioat3_sed_cache); | |
266 | ||
267 | err_ioat2_cache: | |
268 | kmem_cache_destroy(ioat2_cache); | |
162b96e6 DW |
269 | |
270 | return err; | |
8ab89567 SN |
271 | } |
272 | module_init(ioat_init_module); | |
273 | ||
274 | static void __exit ioat_exit_module(void) | |
275 | { | |
7df7cf06 | 276 | pci_unregister_driver(&ioat_pci_driver); |
162b96e6 | 277 | kmem_cache_destroy(ioat2_cache); |
8ab89567 SN |
278 | } |
279 | module_exit(ioat_exit_module); |