Commit | Line | Data |
---|---|---|
0bbd5f4e CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License as published by the Free | |
6 | * Software Foundation; either version 2 of the License, or (at your option) | |
7 | * any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in the | |
19 | * file called COPYING. | |
20 | */ | |
21 | ||
22 | /* | |
23 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
24 | * copy operations. | |
25 | */ | |
26 | ||
27 | #include <linux/init.h> | |
28 | #include <linux/module.h> | |
29 | #include <linux/pci.h> | |
30 | #include <linux/interrupt.h> | |
31 | #include <linux/dmaengine.h> | |
32 | #include <linux/delay.h> | |
6b00c92c | 33 | #include <linux/dma-mapping.h> |
0bbd5f4e | 34 | #include "ioatdma.h" |
0bbd5f4e CL |
35 | #include "ioatdma_registers.h" |
36 | #include "ioatdma_hw.h" | |
37 | ||
38 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | |
39 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | |
40 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | |
7405f74b | 41 | #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx) |
0bbd5f4e CL |
42 | |
43 | /* internal functions */ | |
44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | |
428ed602 | 45 | static void ioat_shutdown(struct pci_dev *pdev); |
0bbd5f4e CL |
46 | static void __devexit ioat_remove(struct pci_dev *pdev); |
47 | ||
48 | static int enumerate_dma_channels(struct ioat_device *device) | |
49 | { | |
50 | u8 xfercap_scale; | |
51 | u32 xfercap; | |
52 | int i; | |
53 | struct ioat_dma_chan *ioat_chan; | |
54 | ||
e3828811 CL |
55 | device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); |
56 | xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); | |
0bbd5f4e CL |
57 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); |
58 | ||
59 | for (i = 0; i < device->common.chancnt; i++) { | |
60 | ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); | |
61 | if (!ioat_chan) { | |
62 | device->common.chancnt = i; | |
63 | break; | |
64 | } | |
65 | ||
66 | ioat_chan->device = device; | |
67 | ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1)); | |
68 | ioat_chan->xfercap = xfercap; | |
69 | spin_lock_init(&ioat_chan->cleanup_lock); | |
70 | spin_lock_init(&ioat_chan->desc_lock); | |
71 | INIT_LIST_HEAD(&ioat_chan->free_desc); | |
72 | INIT_LIST_HEAD(&ioat_chan->used_desc); | |
73 | /* This should be made common somewhere in dmaengine.c */ | |
74 | ioat_chan->common.device = &device->common; | |
0bbd5f4e CL |
75 | list_add_tail(&ioat_chan->common.device_node, |
76 | &device->common.channels); | |
77 | } | |
78 | return device->common.chancnt; | |
79 | } | |
80 | ||
7405f74b DW |
81 | static void |
82 | ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | |
83 | { | |
84 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | |
85 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | |
86 | ||
87 | pci_unmap_addr_set(desc, src, addr); | |
88 | ||
89 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | |
90 | iter->hw->src_addr = addr; | |
91 | addr += ioat_chan->xfercap; | |
92 | } | |
93 | ||
94 | } | |
95 | ||
96 | static void | |
97 | ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | |
98 | { | |
99 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | |
100 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | |
101 | ||
102 | pci_unmap_addr_set(desc, dst, addr); | |
103 | ||
104 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | |
105 | iter->hw->dst_addr = addr; | |
106 | addr += ioat_chan->xfercap; | |
107 | } | |
108 | } | |
109 | ||
110 | static dma_cookie_t | |
111 | ioat_tx_submit(struct dma_async_tx_descriptor *tx) | |
112 | { | |
113 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | |
114 | struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); | |
115 | int append = 0; | |
116 | dma_cookie_t cookie; | |
117 | struct ioat_desc_sw *group_start; | |
118 | ||
119 | group_start = list_entry(desc->async_tx.tx_list.next, | |
120 | struct ioat_desc_sw, node); | |
121 | spin_lock_bh(&ioat_chan->desc_lock); | |
122 | /* cookie incr and addition to used_list must be atomic */ | |
123 | cookie = ioat_chan->common.cookie; | |
124 | cookie++; | |
125 | if (cookie < 0) | |
126 | cookie = 1; | |
127 | ioat_chan->common.cookie = desc->async_tx.cookie = cookie; | |
128 | ||
129 | /* write address into NextDescriptor field of last desc in chain */ | |
130 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = | |
131 | group_start->async_tx.phys; | |
132 | list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev); | |
133 | ||
134 | ioat_chan->pending += desc->tx_cnt; | |
135 | if (ioat_chan->pending >= 4) { | |
136 | append = 1; | |
137 | ioat_chan->pending = 0; | |
138 | } | |
139 | spin_unlock_bh(&ioat_chan->desc_lock); | |
140 | ||
141 | if (append) | |
142 | writeb(IOAT_CHANCMD_APPEND, | |
143 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | |
144 | ||
145 | return cookie; | |
146 | } | |
147 | ||
0bbd5f4e CL |
148 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( |
149 | struct ioat_dma_chan *ioat_chan, | |
47b16539 | 150 | gfp_t flags) |
0bbd5f4e CL |
151 | { |
152 | struct ioat_dma_descriptor *desc; | |
153 | struct ioat_desc_sw *desc_sw; | |
154 | struct ioat_device *ioat_device; | |
155 | dma_addr_t phys; | |
156 | ||
157 | ioat_device = to_ioat_device(ioat_chan->common.device); | |
158 | desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); | |
159 | if (unlikely(!desc)) | |
160 | return NULL; | |
161 | ||
162 | desc_sw = kzalloc(sizeof(*desc_sw), flags); | |
163 | if (unlikely(!desc_sw)) { | |
164 | pci_pool_free(ioat_device->dma_pool, desc, phys); | |
165 | return NULL; | |
166 | } | |
167 | ||
168 | memset(desc, 0, sizeof(*desc)); | |
7405f74b DW |
169 | dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common); |
170 | desc_sw->async_tx.tx_set_src = ioat_set_src; | |
171 | desc_sw->async_tx.tx_set_dest = ioat_set_dest; | |
172 | desc_sw->async_tx.tx_submit = ioat_tx_submit; | |
173 | INIT_LIST_HEAD(&desc_sw->async_tx.tx_list); | |
0bbd5f4e | 174 | desc_sw->hw = desc; |
7405f74b | 175 | desc_sw->async_tx.phys = phys; |
0bbd5f4e CL |
176 | |
177 | return desc_sw; | |
178 | } | |
179 | ||
180 | #define INITIAL_IOAT_DESC_COUNT 128 | |
181 | ||
182 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); | |
183 | ||
184 | /* returns the actual number of allocated descriptors */ | |
185 | static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |
186 | { | |
187 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
188 | struct ioat_desc_sw *desc = NULL; | |
189 | u16 chanctrl; | |
190 | u32 chanerr; | |
191 | int i; | |
192 | LIST_HEAD(tmp_list); | |
193 | ||
194 | /* | |
195 | * In-use bit automatically set by reading chanctrl | |
196 | * If 0, we got it, if 1, someone else did | |
197 | */ | |
e3828811 | 198 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e CL |
199 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) |
200 | return -EBUSY; | |
201 | ||
202 | /* Setup register to interrupt and write completion status on error */ | |
203 | chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE | | |
204 | IOAT_CHANCTRL_ERR_INT_EN | | |
205 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | | |
206 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | |
e3828811 | 207 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e | 208 | |
e3828811 | 209 | chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
0bbd5f4e CL |
210 | if (chanerr) { |
211 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); | |
e3828811 | 212 | writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
0bbd5f4e CL |
213 | } |
214 | ||
215 | /* Allocate descriptors */ | |
216 | for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { | |
217 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
218 | if (!desc) { | |
219 | printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); | |
220 | break; | |
221 | } | |
222 | list_add_tail(&desc->node, &tmp_list); | |
223 | } | |
224 | spin_lock_bh(&ioat_chan->desc_lock); | |
225 | list_splice(&tmp_list, &ioat_chan->free_desc); | |
226 | spin_unlock_bh(&ioat_chan->desc_lock); | |
227 | ||
228 | /* allocate a completion writeback area */ | |
229 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ | |
230 | ioat_chan->completion_virt = | |
231 | pci_pool_alloc(ioat_chan->device->completion_pool, | |
232 | GFP_KERNEL, | |
233 | &ioat_chan->completion_addr); | |
234 | memset(ioat_chan->completion_virt, 0, | |
235 | sizeof(*ioat_chan->completion_virt)); | |
e3828811 CL |
236 | writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF, |
237 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); | |
238 | writel(((u64) ioat_chan->completion_addr) >> 32, | |
239 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); | |
0bbd5f4e CL |
240 | |
241 | ioat_start_null_desc(ioat_chan); | |
242 | return i; | |
243 | } | |
244 | ||
245 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); | |
246 | ||
247 | static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |
248 | { | |
249 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
250 | struct ioat_device *ioat_device = to_ioat_device(chan->device); | |
251 | struct ioat_desc_sw *desc, *_desc; | |
252 | u16 chanctrl; | |
253 | int in_use_descs = 0; | |
254 | ||
255 | ioat_dma_memcpy_cleanup(ioat_chan); | |
256 | ||
e3828811 | 257 | writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
0bbd5f4e CL |
258 | |
259 | spin_lock_bh(&ioat_chan->desc_lock); | |
260 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | |
261 | in_use_descs++; | |
262 | list_del(&desc->node); | |
7405f74b DW |
263 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
264 | desc->async_tx.phys); | |
0bbd5f4e CL |
265 | kfree(desc); |
266 | } | |
267 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | |
268 | list_del(&desc->node); | |
7405f74b DW |
269 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
270 | desc->async_tx.phys); | |
0bbd5f4e CL |
271 | kfree(desc); |
272 | } | |
273 | spin_unlock_bh(&ioat_chan->desc_lock); | |
274 | ||
275 | pci_pool_free(ioat_device->completion_pool, | |
276 | ioat_chan->completion_virt, | |
277 | ioat_chan->completion_addr); | |
278 | ||
279 | /* one is ok since we left it on there on purpose */ | |
280 | if (in_use_descs > 1) | |
281 | printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", | |
282 | in_use_descs - 1); | |
283 | ||
284 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | |
285 | ||
286 | /* Tell hw the chan is free */ | |
e3828811 | 287 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e | 288 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; |
e3828811 | 289 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
0bbd5f4e CL |
290 | } |
291 | ||
7405f74b DW |
292 | static struct dma_async_tx_descriptor * |
293 | ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) | |
0bbd5f4e | 294 | { |
7405f74b DW |
295 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); |
296 | struct ioat_desc_sw *first, *prev, *new; | |
0bbd5f4e CL |
297 | LIST_HEAD(new_chain); |
298 | u32 copy; | |
299 | size_t orig_len; | |
7405f74b | 300 | int desc_count = 0; |
0bbd5f4e CL |
301 | |
302 | if (!len) | |
7405f74b | 303 | return NULL; |
0bbd5f4e CL |
304 | |
305 | orig_len = len; | |
0bbd5f4e CL |
306 | |
307 | first = NULL; | |
308 | prev = NULL; | |
309 | ||
310 | spin_lock_bh(&ioat_chan->desc_lock); | |
0bbd5f4e CL |
311 | while (len) { |
312 | if (!list_empty(&ioat_chan->free_desc)) { | |
313 | new = to_ioat_desc(ioat_chan->free_desc.next); | |
314 | list_del(&new->node); | |
315 | } else { | |
316 | /* try to get another desc */ | |
317 | new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); | |
318 | /* will this ever happen? */ | |
319 | /* TODO add upper limit on these */ | |
320 | BUG_ON(!new); | |
321 | } | |
322 | ||
323 | copy = min((u32) len, ioat_chan->xfercap); | |
324 | ||
325 | new->hw->size = copy; | |
326 | new->hw->ctl = 0; | |
7405f74b DW |
327 | new->async_tx.cookie = 0; |
328 | new->async_tx.ack = 1; | |
0bbd5f4e CL |
329 | |
330 | /* chain together the physical address list for the HW */ | |
331 | if (!first) | |
332 | first = new; | |
333 | else | |
7405f74b | 334 | prev->hw->next = (u64) new->async_tx.phys; |
0bbd5f4e CL |
335 | |
336 | prev = new; | |
0bbd5f4e | 337 | len -= copy; |
0bbd5f4e CL |
338 | list_add_tail(&new->node, &new_chain); |
339 | desc_count++; | |
340 | } | |
0bbd5f4e | 341 | |
7405f74b | 342 | list_splice(&new_chain, &new->async_tx.tx_list); |
0bbd5f4e | 343 | |
7405f74b DW |
344 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; |
345 | new->hw->next = 0; | |
346 | new->tx_cnt = desc_count; | |
347 | new->async_tx.ack = 0; /* client is in control of this ack */ | |
348 | new->async_tx.cookie = -EBUSY; | |
0bbd5f4e | 349 | |
0bbd5f4e CL |
350 | pci_unmap_len_set(new, src_len, orig_len); |
351 | pci_unmap_len_set(new, dst_len, orig_len); | |
0bbd5f4e CL |
352 | spin_unlock_bh(&ioat_chan->desc_lock); |
353 | ||
7405f74b | 354 | return new ? &new->async_tx : NULL; |
0bbd5f4e CL |
355 | } |
356 | ||
0bbd5f4e CL |
357 | |
358 | /** | |
6508871e | 359 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw |
0bbd5f4e CL |
360 | * @chan: DMA channel handle |
361 | */ | |
362 | ||
363 | static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) | |
364 | { | |
365 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
366 | ||
367 | if (ioat_chan->pending != 0) { | |
368 | ioat_chan->pending = 0; | |
e3828811 CL |
369 | writeb(IOAT_CHANCMD_APPEND, |
370 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | |
0bbd5f4e CL |
371 | } |
372 | } | |
373 | ||
374 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |
375 | { | |
376 | unsigned long phys_complete; | |
377 | struct ioat_desc_sw *desc, *_desc; | |
378 | dma_cookie_t cookie = 0; | |
379 | ||
380 | prefetch(chan->completion_virt); | |
381 | ||
382 | if (!spin_trylock(&chan->cleanup_lock)) | |
383 | return; | |
384 | ||
385 | /* The completion writeback can happen at any time, | |
386 | so reads by the driver need to be atomic operations | |
387 | The descriptor physical addresses are limited to 32-bits | |
388 | when the CPU can only do a 32-bit mov */ | |
389 | ||
390 | #if (BITS_PER_LONG == 64) | |
391 | phys_complete = | |
392 | chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | |
393 | #else | |
394 | phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | |
395 | #endif | |
396 | ||
397 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | |
398 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | |
399 | printk("IOAT: Channel halted, chanerr = %x\n", | |
e3828811 | 400 | readl(chan->reg_base + IOAT_CHANERR_OFFSET)); |
0bbd5f4e CL |
401 | |
402 | /* TODO do something to salvage the situation */ | |
403 | } | |
404 | ||
405 | if (phys_complete == chan->last_completion) { | |
406 | spin_unlock(&chan->cleanup_lock); | |
407 | return; | |
408 | } | |
409 | ||
410 | spin_lock_bh(&chan->desc_lock); | |
411 | list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { | |
412 | ||
413 | /* | |
414 | * Incoming DMA requests may use multiple descriptors, due to | |
415 | * exceeding xfercap, perhaps. If so, only the last one will | |
416 | * have a cookie, and require unmapping. | |
417 | */ | |
7405f74b DW |
418 | if (desc->async_tx.cookie) { |
419 | cookie = desc->async_tx.cookie; | |
0bbd5f4e CL |
420 | |
421 | /* yes we are unmapping both _page and _single alloc'd | |
422 | regions with unmap_page. Is this *really* that bad? | |
423 | */ | |
424 | pci_unmap_page(chan->device->pdev, | |
425 | pci_unmap_addr(desc, dst), | |
426 | pci_unmap_len(desc, dst_len), | |
427 | PCI_DMA_FROMDEVICE); | |
428 | pci_unmap_page(chan->device->pdev, | |
429 | pci_unmap_addr(desc, src), | |
430 | pci_unmap_len(desc, src_len), | |
431 | PCI_DMA_TODEVICE); | |
432 | } | |
433 | ||
7405f74b DW |
434 | if (desc->async_tx.phys != phys_complete) { |
435 | /* a completed entry, but not the last, so cleanup | |
436 | * if the client is done with the descriptor | |
437 | */ | |
438 | if (desc->async_tx.ack) { | |
439 | list_del(&desc->node); | |
440 | list_add_tail(&desc->node, &chan->free_desc); | |
441 | } else | |
442 | desc->async_tx.cookie = 0; | |
0bbd5f4e CL |
443 | } else { |
444 | /* last used desc. Do not remove, so we can append from | |
445 | it, but don't look at it next time, either */ | |
7405f74b | 446 | desc->async_tx.cookie = 0; |
0bbd5f4e CL |
447 | |
448 | /* TODO check status bits? */ | |
449 | break; | |
450 | } | |
451 | } | |
452 | ||
453 | spin_unlock_bh(&chan->desc_lock); | |
454 | ||
455 | chan->last_completion = phys_complete; | |
456 | if (cookie != 0) | |
457 | chan->completed_cookie = cookie; | |
458 | ||
459 | spin_unlock(&chan->cleanup_lock); | |
460 | } | |
461 | ||
7405f74b DW |
462 | static void ioat_dma_dependency_added(struct dma_chan *chan) |
463 | { | |
464 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
465 | spin_lock_bh(&ioat_chan->desc_lock); | |
466 | if (ioat_chan->pending == 0) { | |
467 | spin_unlock_bh(&ioat_chan->desc_lock); | |
468 | ioat_dma_memcpy_cleanup(ioat_chan); | |
469 | } else | |
470 | spin_unlock_bh(&ioat_chan->desc_lock); | |
471 | } | |
472 | ||
0bbd5f4e CL |
473 | /** |
474 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | |
475 | * @chan: IOAT DMA channel handle | |
476 | * @cookie: DMA transaction identifier | |
6508871e RD |
477 | * @done: if not %NULL, updated with last completed transaction |
478 | * @used: if not %NULL, updated with last used transaction | |
0bbd5f4e CL |
479 | */ |
480 | ||
481 | static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, | |
482 | dma_cookie_t cookie, | |
483 | dma_cookie_t *done, | |
484 | dma_cookie_t *used) | |
485 | { | |
486 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
487 | dma_cookie_t last_used; | |
488 | dma_cookie_t last_complete; | |
489 | enum dma_status ret; | |
490 | ||
491 | last_used = chan->cookie; | |
492 | last_complete = ioat_chan->completed_cookie; | |
493 | ||
494 | if (done) | |
495 | *done= last_complete; | |
496 | if (used) | |
497 | *used = last_used; | |
498 | ||
499 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
500 | if (ret == DMA_SUCCESS) | |
501 | return ret; | |
502 | ||
503 | ioat_dma_memcpy_cleanup(ioat_chan); | |
504 | ||
505 | last_used = chan->cookie; | |
506 | last_complete = ioat_chan->completed_cookie; | |
507 | ||
508 | if (done) | |
509 | *done= last_complete; | |
510 | if (used) | |
511 | *used = last_used; | |
512 | ||
513 | return dma_async_is_complete(cookie, last_complete, last_used); | |
514 | } | |
515 | ||
516 | /* PCI API */ | |
517 | ||
518 | static struct pci_device_id ioat_pci_tbl[] = { | |
519 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, | |
520 | { 0, } | |
521 | }; | |
522 | ||
92504f79 | 523 | static struct pci_driver ioat_pci_driver = { |
0bbd5f4e CL |
524 | .name = "ioatdma", |
525 | .id_table = ioat_pci_tbl, | |
526 | .probe = ioat_probe, | |
428ed602 | 527 | .shutdown = ioat_shutdown, |
0bbd5f4e CL |
528 | .remove = __devexit_p(ioat_remove), |
529 | }; | |
530 | ||
7d12e780 | 531 | static irqreturn_t ioat_do_interrupt(int irq, void *data) |
0bbd5f4e CL |
532 | { |
533 | struct ioat_device *instance = data; | |
534 | unsigned long attnstatus; | |
535 | u8 intrctrl; | |
536 | ||
e3828811 | 537 | intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
538 | |
539 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | |
540 | return IRQ_NONE; | |
541 | ||
542 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | |
e3828811 | 543 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
544 | return IRQ_NONE; |
545 | } | |
546 | ||
e3828811 | 547 | attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); |
0bbd5f4e CL |
548 | |
549 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); | |
550 | ||
e3828811 | 551 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
552 | return IRQ_HANDLED; |
553 | } | |
554 | ||
555 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |
556 | { | |
557 | struct ioat_desc_sw *desc; | |
558 | ||
559 | spin_lock_bh(&ioat_chan->desc_lock); | |
560 | ||
561 | if (!list_empty(&ioat_chan->free_desc)) { | |
562 | desc = to_ioat_desc(ioat_chan->free_desc.next); | |
563 | list_del(&desc->node); | |
564 | } else { | |
565 | /* try to get another desc */ | |
566 | spin_unlock_bh(&ioat_chan->desc_lock); | |
567 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
568 | spin_lock_bh(&ioat_chan->desc_lock); | |
569 | /* will this ever happen? */ | |
570 | BUG_ON(!desc); | |
571 | } | |
572 | ||
573 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | |
574 | desc->hw->next = 0; | |
7405f74b | 575 | desc->async_tx.ack = 1; |
0bbd5f4e CL |
576 | |
577 | list_add_tail(&desc->node, &ioat_chan->used_desc); | |
578 | spin_unlock_bh(&ioat_chan->desc_lock); | |
579 | ||
7405f74b | 580 | writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF, |
e3828811 | 581 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); |
7405f74b | 582 | writel(((u64) desc->async_tx.phys) >> 32, |
70774b47 CL |
583 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); |
584 | ||
e3828811 | 585 | writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
0bbd5f4e CL |
586 | } |
587 | ||
588 | /* | |
589 | * Perform a IOAT transaction to verify the HW works. | |
590 | */ | |
591 | #define IOAT_TEST_SIZE 2000 | |
592 | ||
593 | static int ioat_self_test(struct ioat_device *device) | |
594 | { | |
595 | int i; | |
596 | u8 *src; | |
597 | u8 *dest; | |
598 | struct dma_chan *dma_chan; | |
7405f74b DW |
599 | struct dma_async_tx_descriptor *tx; |
600 | dma_addr_t addr; | |
0bbd5f4e CL |
601 | dma_cookie_t cookie; |
602 | int err = 0; | |
603 | ||
e94b1766 | 604 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
605 | if (!src) |
606 | return -ENOMEM; | |
e94b1766 | 607 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
0bbd5f4e CL |
608 | if (!dest) { |
609 | kfree(src); | |
610 | return -ENOMEM; | |
611 | } | |
612 | ||
613 | /* Fill in src buffer */ | |
614 | for (i = 0; i < IOAT_TEST_SIZE; i++) | |
615 | src[i] = (u8)i; | |
616 | ||
617 | /* Start copy, using first DMA channel */ | |
618 | dma_chan = container_of(device->common.channels.next, | |
619 | struct dma_chan, | |
620 | device_node); | |
621 | if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { | |
622 | err = -ENODEV; | |
623 | goto out; | |
624 | } | |
625 | ||
7405f74b DW |
626 | tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0); |
627 | async_tx_ack(tx); | |
628 | addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE, | |
629 | DMA_TO_DEVICE); | |
630 | ioat_set_src(addr, tx, 0); | |
631 | addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE, | |
632 | DMA_FROM_DEVICE); | |
633 | ioat_set_dest(addr, tx, 0); | |
634 | cookie = ioat_tx_submit(tx); | |
0bbd5f4e CL |
635 | ioat_dma_memcpy_issue_pending(dma_chan); |
636 | msleep(1); | |
637 | ||
638 | if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { | |
639 | printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); | |
640 | err = -ENODEV; | |
641 | goto free_resources; | |
642 | } | |
643 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { | |
644 | printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); | |
645 | err = -ENODEV; | |
646 | goto free_resources; | |
647 | } | |
648 | ||
649 | free_resources: | |
650 | ioat_dma_free_chan_resources(dma_chan); | |
651 | out: | |
652 | kfree(src); | |
653 | kfree(dest); | |
654 | return err; | |
655 | } | |
656 | ||
657 | static int __devinit ioat_probe(struct pci_dev *pdev, | |
658 | const struct pci_device_id *ent) | |
659 | { | |
660 | int err; | |
661 | unsigned long mmio_start, mmio_len; | |
47b16539 | 662 | void __iomem *reg_base; |
0bbd5f4e CL |
663 | struct ioat_device *device; |
664 | ||
665 | err = pci_enable_device(pdev); | |
666 | if (err) | |
667 | goto err_enable_device; | |
668 | ||
669 | err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); | |
670 | if (err) | |
671 | err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); | |
672 | if (err) | |
673 | goto err_set_dma_mask; | |
674 | ||
92504f79 | 675 | err = pci_request_regions(pdev, ioat_pci_driver.name); |
0bbd5f4e CL |
676 | if (err) |
677 | goto err_request_regions; | |
678 | ||
679 | mmio_start = pci_resource_start(pdev, 0); | |
680 | mmio_len = pci_resource_len(pdev, 0); | |
681 | ||
682 | reg_base = ioremap(mmio_start, mmio_len); | |
683 | if (!reg_base) { | |
684 | err = -ENOMEM; | |
685 | goto err_ioremap; | |
686 | } | |
687 | ||
688 | device = kzalloc(sizeof(*device), GFP_KERNEL); | |
689 | if (!device) { | |
690 | err = -ENOMEM; | |
691 | goto err_kzalloc; | |
692 | } | |
693 | ||
694 | /* DMA coherent memory pool for DMA descriptor allocations */ | |
695 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, | |
696 | sizeof(struct ioat_dma_descriptor), 64, 0); | |
697 | if (!device->dma_pool) { | |
698 | err = -ENOMEM; | |
699 | goto err_dma_pool; | |
700 | } | |
701 | ||
702 | device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); | |
703 | if (!device->completion_pool) { | |
704 | err = -ENOMEM; | |
705 | goto err_completion_pool; | |
706 | } | |
707 | ||
708 | device->pdev = pdev; | |
709 | pci_set_drvdata(pdev, device); | |
710 | #ifdef CONFIG_PCI_MSI | |
711 | if (pci_enable_msi(pdev) == 0) { | |
712 | device->msi = 1; | |
713 | } else { | |
714 | device->msi = 0; | |
715 | } | |
716 | #endif | |
dace1453 | 717 | err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", |
0bbd5f4e CL |
718 | device); |
719 | if (err) | |
720 | goto err_irq; | |
721 | ||
722 | device->reg_base = reg_base; | |
723 | ||
e3828811 | 724 | writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET); |
0bbd5f4e CL |
725 | pci_set_master(pdev); |
726 | ||
727 | INIT_LIST_HEAD(&device->common.channels); | |
728 | enumerate_dma_channels(device); | |
729 | ||
7405f74b | 730 | dma_cap_set(DMA_MEMCPY, device->common.cap_mask); |
0bbd5f4e CL |
731 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; |
732 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | |
7405f74b DW |
733 | device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy; |
734 | device->common.device_is_tx_complete = ioat_dma_is_complete; | |
735 | device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; | |
736 | device->common.device_dependency_added = ioat_dma_dependency_added; | |
737 | device->common.dev = &pdev->dev; | |
0bbd5f4e CL |
738 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", |
739 | device->common.chancnt); | |
740 | ||
741 | err = ioat_self_test(device); | |
742 | if (err) | |
743 | goto err_self_test; | |
744 | ||
745 | dma_async_device_register(&device->common); | |
746 | ||
747 | return 0; | |
748 | ||
749 | err_self_test: | |
750 | err_irq: | |
751 | pci_pool_destroy(device->completion_pool); | |
752 | err_completion_pool: | |
753 | pci_pool_destroy(device->dma_pool); | |
754 | err_dma_pool: | |
755 | kfree(device); | |
756 | err_kzalloc: | |
757 | iounmap(reg_base); | |
758 | err_ioremap: | |
759 | pci_release_regions(pdev); | |
760 | err_request_regions: | |
761 | err_set_dma_mask: | |
762 | pci_disable_device(pdev); | |
763 | err_enable_device: | |
428ed602 DA |
764 | |
765 | printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n"); | |
766 | ||
0bbd5f4e CL |
767 | return err; |
768 | } | |
769 | ||
428ed602 DA |
770 | static void ioat_shutdown(struct pci_dev *pdev) |
771 | { | |
772 | struct ioat_device *device; | |
773 | device = pci_get_drvdata(pdev); | |
774 | ||
775 | dma_async_device_unregister(&device->common); | |
776 | } | |
777 | ||
0bbd5f4e CL |
778 | static void __devexit ioat_remove(struct pci_dev *pdev) |
779 | { | |
780 | struct ioat_device *device; | |
781 | struct dma_chan *chan, *_chan; | |
782 | struct ioat_dma_chan *ioat_chan; | |
783 | ||
784 | device = pci_get_drvdata(pdev); | |
785 | dma_async_device_unregister(&device->common); | |
786 | ||
787 | free_irq(device->pdev->irq, device); | |
788 | #ifdef CONFIG_PCI_MSI | |
789 | if (device->msi) | |
790 | pci_disable_msi(device->pdev); | |
791 | #endif | |
792 | pci_pool_destroy(device->dma_pool); | |
793 | pci_pool_destroy(device->completion_pool); | |
794 | iounmap(device->reg_base); | |
795 | pci_release_regions(pdev); | |
796 | pci_disable_device(pdev); | |
797 | list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { | |
798 | ioat_chan = to_ioat_chan(chan); | |
799 | list_del(&chan->device_node); | |
800 | kfree(ioat_chan); | |
801 | } | |
802 | kfree(device); | |
803 | } | |
804 | ||
805 | /* MODULE API */ | |
000725d5 | 806 | MODULE_VERSION("1.9"); |
0bbd5f4e CL |
807 | MODULE_LICENSE("GPL"); |
808 | MODULE_AUTHOR("Intel Corporation"); | |
809 | ||
810 | static int __init ioat_init_module(void) | |
811 | { | |
812 | /* it's currently unsafe to unload this module */ | |
813 | /* if forced, worst case is that rmmod hangs */ | |
8070b2b1 | 814 | __unsafe(THIS_MODULE); |
0bbd5f4e | 815 | |
92504f79 | 816 | return pci_register_driver(&ioat_pci_driver); |
0bbd5f4e CL |
817 | } |
818 | ||
819 | module_init(ioat_init_module); | |
820 | ||
821 | static void __exit ioat_exit_module(void) | |
822 | { | |
92504f79 | 823 | pci_unregister_driver(&ioat_pci_driver); |
0bbd5f4e CL |
824 | } |
825 | ||
826 | module_exit(ioat_exit_module); |