2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
20 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
21 * does asynchronous data movement and checksumming operations.
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/pci.h>
28 #include <linux/interrupt.h>
29 #include <linux/dmaengine.h>
30 #include <linux/delay.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/workqueue.h>
33 #include <linux/prefetch.h>
36 #include "registers.h"
39 #include "../dmaengine.h"
41 int ioat_ring_alloc_order
= 8;
42 module_param(ioat_ring_alloc_order
, int, 0644);
43 MODULE_PARM_DESC(ioat_ring_alloc_order
,
44 "ioat2+: allocate 2^n descriptors per channel"
45 " (default: 8 max: 16)");
46 static int ioat_ring_max_alloc_order
= IOAT_MAX_ORDER
;
47 module_param(ioat_ring_max_alloc_order
, int, 0644);
48 MODULE_PARM_DESC(ioat_ring_max_alloc_order
,
49 "ioat2+: upper limit for ring size (default: 16)");
51 void __ioat2_issue_pending(struct ioat2_dma_chan
*ioat
)
53 struct ioat_chan_common
*chan
= &ioat
->base
;
55 ioat
->dmacount
+= ioat2_ring_pending(ioat
);
56 ioat
->issued
= ioat
->head
;
57 writew(ioat
->dmacount
, chan
->reg_base
+ IOAT_CHAN_DMACOUNT_OFFSET
);
59 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
60 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
63 void ioat2_issue_pending(struct dma_chan
*c
)
65 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
67 if (ioat2_ring_pending(ioat
)) {
68 spin_lock_bh(&ioat
->prep_lock
);
69 __ioat2_issue_pending(ioat
);
70 spin_unlock_bh(&ioat
->prep_lock
);
75 * ioat2_update_pending - log pending descriptors
76 * @ioat: ioat2+ channel
78 * Check if the number of unsubmitted descriptors has exceeded the
79 * watermark. Called with prep_lock held
81 static void ioat2_update_pending(struct ioat2_dma_chan
*ioat
)
83 if (ioat2_ring_pending(ioat
) > ioat_pending_level
)
84 __ioat2_issue_pending(ioat
);
87 static void __ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
89 struct ioat_ring_ent
*desc
;
90 struct ioat_dma_descriptor
*hw
;
92 if (ioat2_ring_space(ioat
) < 1) {
93 dev_err(to_dev(&ioat
->base
),
94 "Unable to start null desc - ring full\n");
98 dev_dbg(to_dev(&ioat
->base
), "%s: head: %#x tail: %#x issued: %#x\n",
99 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
100 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
);
105 hw
->ctl_f
.int_en
= 1;
106 hw
->ctl_f
.compl_write
= 1;
107 /* set size to non-zero value (channel returns error when size is 0) */
108 hw
->size
= NULL_DESC_BUFFER_SIZE
;
111 async_tx_ack(&desc
->txd
);
112 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
113 dump_desc_dbg(ioat
, desc
);
116 __ioat2_issue_pending(ioat
);
119 static void ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
121 spin_lock_bh(&ioat
->prep_lock
);
122 __ioat2_start_null_desc(ioat
);
123 spin_unlock_bh(&ioat
->prep_lock
);
126 void __ioat2_restart_chan(struct ioat2_dma_chan
*ioat
)
128 struct ioat_chan_common
*chan
= &ioat
->base
;
130 /* set the tail to be re-issued */
131 ioat
->issued
= ioat
->tail
;
133 set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
134 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
136 dev_dbg(to_dev(chan
),
137 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
138 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
140 if (ioat2_ring_pending(ioat
)) {
141 struct ioat_ring_ent
*desc
;
143 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
);
144 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
145 __ioat2_issue_pending(ioat
);
147 __ioat2_start_null_desc(ioat
);
150 int ioat2_quiesce(struct ioat_chan_common
*chan
, unsigned long tmo
)
152 unsigned long end
= jiffies
+ tmo
;
156 status
= ioat_chansts(chan
);
157 if (is_ioat_active(status
) || is_ioat_idle(status
))
159 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
160 if (tmo
&& time_after(jiffies
, end
)) {
164 status
= ioat_chansts(chan
);
171 int ioat2_reset_sync(struct ioat_chan_common
*chan
, unsigned long tmo
)
173 unsigned long end
= jiffies
+ tmo
;
177 while (ioat_reset_pending(chan
)) {
178 if (end
&& time_after(jiffies
, end
)) {
189 * ioat2_enumerate_channels - find and initialize the device's channels
190 * @device: the device to be enumerated
192 int ioat2_enumerate_channels(struct ioatdma_device
*device
)
194 struct ioat2_dma_chan
*ioat
;
195 struct device
*dev
= &device
->pdev
->dev
;
196 struct dma_device
*dma
= &device
->common
;
200 INIT_LIST_HEAD(&dma
->channels
);
201 dma
->chancnt
= readb(device
->reg_base
+ IOAT_CHANCNT_OFFSET
);
202 dma
->chancnt
&= 0x1f; /* bits [4:0] valid */
203 if (dma
->chancnt
> ARRAY_SIZE(device
->idx
)) {
204 dev_warn(dev
, "(%d) exceeds max supported channels (%zu)\n",
205 dma
->chancnt
, ARRAY_SIZE(device
->idx
));
206 dma
->chancnt
= ARRAY_SIZE(device
->idx
);
208 xfercap_log
= readb(device
->reg_base
+ IOAT_XFERCAP_OFFSET
);
209 xfercap_log
&= 0x1f; /* bits [4:0] valid */
210 if (xfercap_log
== 0)
212 dev_dbg(dev
, "%s: xfercap = %d\n", __func__
, 1 << xfercap_log
);
214 for (i
= 0; i
< dma
->chancnt
; i
++) {
215 ioat
= devm_kzalloc(dev
, sizeof(*ioat
), GFP_KERNEL
);
219 ioat_init_channel(device
, &ioat
->base
, i
);
220 ioat
->xfercap_log
= xfercap_log
;
221 spin_lock_init(&ioat
->prep_lock
);
222 if (device
->reset_hw(&ioat
->base
)) {
231 static dma_cookie_t
ioat2_tx_submit_unlock(struct dma_async_tx_descriptor
*tx
)
233 struct dma_chan
*c
= tx
->chan
;
234 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
235 struct ioat_chan_common
*chan
= &ioat
->base
;
238 cookie
= dma_cookie_assign(tx
);
239 dev_dbg(to_dev(&ioat
->base
), "%s: cookie: %d\n", __func__
, cookie
);
241 if (!test_and_set_bit(IOAT_CHAN_ACTIVE
, &chan
->state
))
242 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
244 /* make descriptor updates visible before advancing ioat->head,
245 * this is purposefully not smp_wmb() since we are also
246 * publishing the descriptor updates to a dma device
250 ioat
->head
+= ioat
->produce
;
252 ioat2_update_pending(ioat
);
253 spin_unlock_bh(&ioat
->prep_lock
);
258 static struct ioat_ring_ent
*ioat2_alloc_ring_ent(struct dma_chan
*chan
, gfp_t flags
)
260 struct ioat_dma_descriptor
*hw
;
261 struct ioat_ring_ent
*desc
;
262 struct ioatdma_device
*dma
;
265 dma
= to_ioatdma_device(chan
->device
);
266 hw
= pci_pool_alloc(dma
->dma_pool
, flags
, &phys
);
269 memset(hw
, 0, sizeof(*hw
));
271 desc
= kmem_cache_zalloc(ioat2_cache
, flags
);
273 pci_pool_free(dma
->dma_pool
, hw
, phys
);
277 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
278 desc
->txd
.tx_submit
= ioat2_tx_submit_unlock
;
280 desc
->txd
.phys
= phys
;
284 static void ioat2_free_ring_ent(struct ioat_ring_ent
*desc
, struct dma_chan
*chan
)
286 struct ioatdma_device
*dma
;
288 dma
= to_ioatdma_device(chan
->device
);
289 pci_pool_free(dma
->dma_pool
, desc
->hw
, desc
->txd
.phys
);
290 kmem_cache_free(ioat2_cache
, desc
);
293 static struct ioat_ring_ent
**ioat2_alloc_ring(struct dma_chan
*c
, int order
, gfp_t flags
)
295 struct ioat_ring_ent
**ring
;
296 int descs
= 1 << order
;
299 if (order
> ioat_get_max_alloc_order())
302 /* allocate the array to hold the software ring */
303 ring
= kcalloc(descs
, sizeof(*ring
), flags
);
306 for (i
= 0; i
< descs
; i
++) {
307 ring
[i
] = ioat2_alloc_ring_ent(c
, flags
);
310 ioat2_free_ring_ent(ring
[i
], c
);
314 set_desc_id(ring
[i
], i
);
318 for (i
= 0; i
< descs
-1; i
++) {
319 struct ioat_ring_ent
*next
= ring
[i
+1];
320 struct ioat_dma_descriptor
*hw
= ring
[i
]->hw
;
322 hw
->next
= next
->txd
.phys
;
324 ring
[i
]->hw
->next
= ring
[0]->txd
.phys
;
329 void ioat2_free_chan_resources(struct dma_chan
*c
);
331 /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
332 * @chan: channel to be initialized
334 int ioat2_alloc_chan_resources(struct dma_chan
*c
)
336 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
337 struct ioat_chan_common
*chan
= &ioat
->base
;
338 struct ioat_ring_ent
**ring
;
343 /* have we already been set up? */
345 return 1 << ioat
->alloc_order
;
347 /* Setup register to interrupt and write completion status on error */
348 writew(IOAT_CHANCTRL_RUN
, chan
->reg_base
+ IOAT_CHANCTRL_OFFSET
);
350 /* allocate a completion writeback area */
351 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
352 chan
->completion
= pci_pool_alloc(chan
->device
->completion_pool
,
353 GFP_KERNEL
, &chan
->completion_dma
);
354 if (!chan
->completion
)
357 memset(chan
->completion
, 0, sizeof(*chan
->completion
));
358 writel(((u64
) chan
->completion_dma
) & 0x00000000FFFFFFFF,
359 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_LOW
);
360 writel(((u64
) chan
->completion_dma
) >> 32,
361 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_HIGH
);
363 order
= ioat_get_alloc_order();
364 ring
= ioat2_alloc_ring(c
, order
, GFP_KERNEL
);
368 spin_lock_bh(&chan
->cleanup_lock
);
369 spin_lock_bh(&ioat
->prep_lock
);
374 ioat
->alloc_order
= order
;
375 set_bit(IOAT_RUN
, &chan
->state
);
376 spin_unlock_bh(&ioat
->prep_lock
);
377 spin_unlock_bh(&chan
->cleanup_lock
);
379 ioat2_start_null_desc(ioat
);
381 /* check that we got off the ground */
384 status
= ioat_chansts(chan
);
385 } while (i
++ < 20 && !is_ioat_active(status
) && !is_ioat_idle(status
));
387 if (is_ioat_active(status
) || is_ioat_idle(status
)) {
388 return 1 << ioat
->alloc_order
;
390 u32 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
392 dev_WARN(to_dev(chan
),
393 "failed to start channel chanerr: %#x\n", chanerr
);
394 ioat2_free_chan_resources(c
);
399 bool reshape_ring(struct ioat2_dma_chan
*ioat
, int order
)
401 /* reshape differs from normal ring allocation in that we want
402 * to allocate a new software ring while only
403 * extending/truncating the hardware ring
405 struct ioat_chan_common
*chan
= &ioat
->base
;
406 struct dma_chan
*c
= &chan
->common
;
407 const u32 curr_size
= ioat2_ring_size(ioat
);
408 const u16 active
= ioat2_ring_active(ioat
);
409 const u32 new_size
= 1 << order
;
410 struct ioat_ring_ent
**ring
;
413 if (order
> ioat_get_max_alloc_order())
416 /* double check that we have at least 1 free descriptor */
417 if (active
== curr_size
)
420 /* when shrinking, verify that we can hold the current active
421 * set in the new ring
423 if (active
>= new_size
)
426 /* allocate the array to hold the software ring */
427 ring
= kcalloc(new_size
, sizeof(*ring
), GFP_NOWAIT
);
431 /* allocate/trim descriptors as needed */
432 if (new_size
> curr_size
) {
433 /* copy current descriptors to the new ring */
434 for (i
= 0; i
< curr_size
; i
++) {
435 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
436 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
438 ring
[new_idx
] = ioat
->ring
[curr_idx
];
439 set_desc_id(ring
[new_idx
], new_idx
);
442 /* add new descriptors to the ring */
443 for (i
= curr_size
; i
< new_size
; i
++) {
444 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
446 ring
[new_idx
] = ioat2_alloc_ring_ent(c
, GFP_NOWAIT
);
447 if (!ring
[new_idx
]) {
449 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
451 ioat2_free_ring_ent(ring
[new_idx
], c
);
456 set_desc_id(ring
[new_idx
], new_idx
);
459 /* hw link new descriptors */
460 for (i
= curr_size
-1; i
< new_size
; i
++) {
461 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
462 struct ioat_ring_ent
*next
= ring
[(new_idx
+1) & (new_size
-1)];
463 struct ioat_dma_descriptor
*hw
= ring
[new_idx
]->hw
;
465 hw
->next
= next
->txd
.phys
;
468 struct ioat_dma_descriptor
*hw
;
469 struct ioat_ring_ent
*next
;
471 /* copy current descriptors to the new ring, dropping the
472 * removed descriptors
474 for (i
= 0; i
< new_size
; i
++) {
475 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
476 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
478 ring
[new_idx
] = ioat
->ring
[curr_idx
];
479 set_desc_id(ring
[new_idx
], new_idx
);
482 /* free deleted descriptors */
483 for (i
= new_size
; i
< curr_size
; i
++) {
484 struct ioat_ring_ent
*ent
;
486 ent
= ioat2_get_ring_ent(ioat
, ioat
->tail
+i
);
487 ioat2_free_ring_ent(ent
, c
);
490 /* fix up hardware ring */
491 hw
= ring
[(ioat
->tail
+new_size
-1) & (new_size
-1)]->hw
;
492 next
= ring
[(ioat
->tail
+new_size
) & (new_size
-1)];
493 hw
->next
= next
->txd
.phys
;
496 dev_dbg(to_dev(chan
), "%s: allocated %d descriptors\n",
501 ioat
->alloc_order
= order
;
507 * ioat2_check_space_lock - verify space and grab ring producer lock
508 * @ioat: ioat2,3 channel (ring) to operate on
509 * @num_descs: allocation length
511 int ioat2_check_space_lock(struct ioat2_dma_chan
*ioat
, int num_descs
)
513 struct ioat_chan_common
*chan
= &ioat
->base
;
517 spin_lock_bh(&ioat
->prep_lock
);
518 /* never allow the last descriptor to be consumed, we need at
519 * least one free at all times to allow for on-the-fly ring
522 if (likely(ioat2_ring_space(ioat
) > num_descs
)) {
523 dev_dbg(to_dev(chan
), "%s: num_descs: %d (%x:%x:%x)\n",
524 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
525 ioat
->produce
= num_descs
;
526 return 0; /* with ioat->prep_lock held */
528 retry
= test_and_set_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
529 spin_unlock_bh(&ioat
->prep_lock
);
531 /* is another cpu already trying to expand the ring? */
535 spin_lock_bh(&chan
->cleanup_lock
);
536 spin_lock_bh(&ioat
->prep_lock
);
537 retry
= reshape_ring(ioat
, ioat
->alloc_order
+ 1);
538 clear_bit(IOAT_RESHAPE_PENDING
, &chan
->state
);
539 spin_unlock_bh(&ioat
->prep_lock
);
540 spin_unlock_bh(&chan
->cleanup_lock
);
542 /* if we were able to expand the ring retry the allocation */
546 if (printk_ratelimit())
547 dev_dbg(to_dev(chan
), "%s: ring full! num_descs: %d (%x:%x:%x)\n",
548 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
550 /* progress reclaim in the allocation failure case we may be
551 * called under bh_disabled so we need to trigger the timer
554 if (time_is_before_jiffies(chan
->timer
.expires
)
555 && timer_pending(&chan
->timer
)) {
556 struct ioatdma_device
*device
= chan
->device
;
558 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
559 device
->timer_fn((unsigned long) &chan
->common
);
565 struct dma_async_tx_descriptor
*
566 ioat2_dma_prep_memcpy_lock(struct dma_chan
*c
, dma_addr_t dma_dest
,
567 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
569 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
570 struct ioat_dma_descriptor
*hw
;
571 struct ioat_ring_ent
*desc
;
572 dma_addr_t dst
= dma_dest
;
573 dma_addr_t src
= dma_src
;
574 size_t total_len
= len
;
575 int num_descs
, idx
, i
;
577 num_descs
= ioat2_xferlen_to_descs(ioat
, len
);
578 if (likely(num_descs
) && ioat2_check_space_lock(ioat
, num_descs
) == 0)
584 size_t copy
= min_t(size_t, len
, 1 << ioat
->xfercap_log
);
586 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
597 dump_desc_dbg(ioat
, desc
);
598 } while (++i
< num_descs
);
600 desc
->txd
.flags
= flags
;
601 desc
->len
= total_len
;
602 hw
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
603 hw
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
604 hw
->ctl_f
.compl_write
= 1;
605 dump_desc_dbg(ioat
, desc
);
606 /* we leave the channel locked to ensure in order submission */
612 * ioat2_free_chan_resources - release all the descriptors
613 * @chan: the channel to be cleaned
615 void ioat2_free_chan_resources(struct dma_chan
*c
)
617 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
618 struct ioat_chan_common
*chan
= &ioat
->base
;
619 struct ioatdma_device
*device
= chan
->device
;
620 struct ioat_ring_ent
*desc
;
621 const int total_descs
= 1 << ioat
->alloc_order
;
625 /* Before freeing channel resources first check
626 * if they have been previously allocated for this channel.
632 device
->reset_hw(chan
);
634 spin_lock_bh(&chan
->cleanup_lock
);
635 spin_lock_bh(&ioat
->prep_lock
);
636 descs
= ioat2_ring_space(ioat
);
637 dev_dbg(to_dev(chan
), "freeing %d idle descriptors\n", descs
);
638 for (i
= 0; i
< descs
; i
++) {
639 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
+ i
);
640 ioat2_free_ring_ent(desc
, c
);
643 if (descs
< total_descs
)
644 dev_err(to_dev(chan
), "Freeing %d in use descriptors!\n",
645 total_descs
- descs
);
647 for (i
= 0; i
< total_descs
- descs
; i
++) {
648 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
649 dump_desc_dbg(ioat
, desc
);
650 ioat2_free_ring_ent(desc
, c
);
655 ioat
->alloc_order
= 0;
656 pci_pool_free(device
->completion_pool
, chan
->completion
,
657 chan
->completion_dma
);
658 spin_unlock_bh(&ioat
->prep_lock
);
659 spin_unlock_bh(&chan
->cleanup_lock
);
661 chan
->last_completion
= 0;
662 chan
->completion_dma
= 0;
666 static ssize_t
ring_size_show(struct dma_chan
*c
, char *page
)
668 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
670 return sprintf(page
, "%d\n", (1 << ioat
->alloc_order
) & ~1);
672 static struct ioat_sysfs_entry ring_size_attr
= __ATTR_RO(ring_size
);
674 static ssize_t
ring_active_show(struct dma_chan
*c
, char *page
)
676 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
678 /* ...taken outside the lock, no need to be precise */
679 return sprintf(page
, "%d\n", ioat2_ring_active(ioat
));
681 static struct ioat_sysfs_entry ring_active_attr
= __ATTR_RO(ring_active
);
683 static struct attribute
*ioat2_attrs
[] = {
684 &ring_size_attr
.attr
,
685 &ring_active_attr
.attr
,
687 &ioat_version_attr
.attr
,
691 struct kobj_type ioat2_ktype
= {
692 .sysfs_ops
= &ioat_sysfs_ops
,
693 .default_attrs
= ioat2_attrs
,