2 * offload engine driver for the Marvell XOR engine
3 * Copyright (C) 2007, 2008, Marvell International Ltd.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 #include <linux/init.h>
16 #include <linux/slab.h>
17 #include <linux/delay.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/of_device.h>
22 #include <linux/platform_device.h>
23 #include <linux/memory.h>
24 #include <linux/clk.h>
26 #include <linux/of_irq.h>
27 #include <linux/irqdomain.h>
28 #include <linux/cpumask.h>
29 #include <linux/platform_data/dma-mv_xor.h>
31 #include "dmaengine.h"
45 static void mv_xor_issue_pending(struct dma_chan
*chan
);
47 #define to_mv_xor_chan(chan) \
48 container_of(chan, struct mv_xor_chan, dmachan)
50 #define to_mv_xor_slot(tx) \
51 container_of(tx, struct mv_xor_desc_slot, async_tx)
53 #define mv_chan_to_devp(chan) \
56 static void mv_desc_init(struct mv_xor_desc_slot
*desc
,
57 dma_addr_t addr
, u32 byte_count
,
58 enum dma_ctrl_flags flags
)
60 struct mv_xor_desc
*hw_desc
= desc
->hw_desc
;
62 hw_desc
->status
= XOR_DESC_DMA_OWNED
;
63 hw_desc
->phy_next_desc
= 0;
64 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
65 hw_desc
->desc_command
= (flags
& DMA_PREP_INTERRUPT
) ?
66 XOR_DESC_EOD_INT_EN
: 0;
67 hw_desc
->phy_dest_addr
= addr
;
68 hw_desc
->byte_count
= byte_count
;
71 static void mv_desc_set_mode(struct mv_xor_desc_slot
*desc
)
73 struct mv_xor_desc
*hw_desc
= desc
->hw_desc
;
78 hw_desc
->desc_command
|= XOR_DESC_OPERATION_XOR
;
81 hw_desc
->desc_command
|= XOR_DESC_OPERATION_MEMCPY
;
89 static void mv_desc_set_next_desc(struct mv_xor_desc_slot
*desc
,
92 struct mv_xor_desc
*hw_desc
= desc
->hw_desc
;
93 BUG_ON(hw_desc
->phy_next_desc
);
94 hw_desc
->phy_next_desc
= next_desc_addr
;
97 static void mv_desc_set_src_addr(struct mv_xor_desc_slot
*desc
,
98 int index
, dma_addr_t addr
)
100 struct mv_xor_desc
*hw_desc
= desc
->hw_desc
;
101 hw_desc
->phy_src_addr
[mv_phy_src_idx(index
)] = addr
;
102 if (desc
->type
== DMA_XOR
)
103 hw_desc
->desc_command
|= (1 << index
);
106 static u32
mv_chan_get_current_desc(struct mv_xor_chan
*chan
)
108 return readl_relaxed(XOR_CURR_DESC(chan
));
111 static void mv_chan_set_next_descriptor(struct mv_xor_chan
*chan
,
114 writel_relaxed(next_desc_addr
, XOR_NEXT_DESC(chan
));
117 static void mv_chan_unmask_interrupts(struct mv_xor_chan
*chan
)
119 u32 val
= readl_relaxed(XOR_INTR_MASK(chan
));
120 val
|= XOR_INTR_MASK_VALUE
<< (chan
->idx
* 16);
121 writel_relaxed(val
, XOR_INTR_MASK(chan
));
124 static u32
mv_chan_get_intr_cause(struct mv_xor_chan
*chan
)
126 u32 intr_cause
= readl_relaxed(XOR_INTR_CAUSE(chan
));
127 intr_cause
= (intr_cause
>> (chan
->idx
* 16)) & 0xFFFF;
131 static void mv_chan_clear_eoc_cause(struct mv_xor_chan
*chan
)
135 val
= XOR_INT_END_OF_DESC
| XOR_INT_END_OF_CHAIN
| XOR_INT_STOPPED
;
136 val
= ~(val
<< (chan
->idx
* 16));
137 dev_dbg(mv_chan_to_devp(chan
), "%s, val 0x%08x\n", __func__
, val
);
138 writel_relaxed(val
, XOR_INTR_CAUSE(chan
));
141 static void mv_chan_clear_err_status(struct mv_xor_chan
*chan
)
143 u32 val
= 0xFFFF0000 >> (chan
->idx
* 16);
144 writel_relaxed(val
, XOR_INTR_CAUSE(chan
));
147 static void mv_chan_set_mode(struct mv_xor_chan
*chan
,
150 u32 config
= readl_relaxed(XOR_CONFIG(chan
));
155 #if defined(__BIG_ENDIAN)
156 config
|= XOR_DESCRIPTOR_SWAP
;
158 config
&= ~XOR_DESCRIPTOR_SWAP
;
161 writel_relaxed(config
, XOR_CONFIG(chan
));
164 static void mv_chan_activate(struct mv_xor_chan
*chan
)
166 dev_dbg(mv_chan_to_devp(chan
), " activate chan.\n");
168 /* writel ensures all descriptors are flushed before activation */
169 writel(BIT(0), XOR_ACTIVATION(chan
));
172 static char mv_chan_is_busy(struct mv_xor_chan
*chan
)
174 u32 state
= readl_relaxed(XOR_ACTIVATION(chan
));
176 state
= (state
>> 4) & 0x3;
178 return (state
== 1) ? 1 : 0;
182 * mv_chan_start_new_chain - program the engine to operate on new
183 * chain headed by sw_desc
184 * Caller must hold &mv_chan->lock while calling this function
186 static void mv_chan_start_new_chain(struct mv_xor_chan
*mv_chan
,
187 struct mv_xor_desc_slot
*sw_desc
)
189 dev_dbg(mv_chan_to_devp(mv_chan
), "%s %d: sw_desc %p\n",
190 __func__
, __LINE__
, sw_desc
);
192 /* set the hardware chain */
193 mv_chan_set_next_descriptor(mv_chan
, sw_desc
->async_tx
.phys
);
196 mv_xor_issue_pending(&mv_chan
->dmachan
);
200 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot
*desc
,
201 struct mv_xor_chan
*mv_chan
,
204 BUG_ON(desc
->async_tx
.cookie
< 0);
206 if (desc
->async_tx
.cookie
> 0) {
207 cookie
= desc
->async_tx
.cookie
;
209 dma_descriptor_unmap(&desc
->async_tx
);
210 /* call the callback (must not sleep or submit new
211 * operations to this channel)
213 dmaengine_desc_get_callback_invoke(&desc
->async_tx
, NULL
);
216 /* run dependent operations */
217 dma_run_dependencies(&desc
->async_tx
);
223 mv_chan_clean_completed_slots(struct mv_xor_chan
*mv_chan
)
225 struct mv_xor_desc_slot
*iter
, *_iter
;
227 dev_dbg(mv_chan_to_devp(mv_chan
), "%s %d\n", __func__
, __LINE__
);
228 list_for_each_entry_safe(iter
, _iter
, &mv_chan
->completed_slots
,
231 if (async_tx_test_ack(&iter
->async_tx
))
232 list_move_tail(&iter
->node
, &mv_chan
->free_slots
);
238 mv_desc_clean_slot(struct mv_xor_desc_slot
*desc
,
239 struct mv_xor_chan
*mv_chan
)
241 dev_dbg(mv_chan_to_devp(mv_chan
), "%s %d: desc %p flags %d\n",
242 __func__
, __LINE__
, desc
, desc
->async_tx
.flags
);
244 /* the client is allowed to attach dependent operations
247 if (!async_tx_test_ack(&desc
->async_tx
))
248 /* move this slot to the completed_slots */
249 list_move_tail(&desc
->node
, &mv_chan
->completed_slots
);
251 list_move_tail(&desc
->node
, &mv_chan
->free_slots
);
256 /* This function must be called with the mv_xor_chan spinlock held */
257 static void mv_chan_slot_cleanup(struct mv_xor_chan
*mv_chan
)
259 struct mv_xor_desc_slot
*iter
, *_iter
;
260 dma_cookie_t cookie
= 0;
261 int busy
= mv_chan_is_busy(mv_chan
);
262 u32 current_desc
= mv_chan_get_current_desc(mv_chan
);
263 int current_cleaned
= 0;
264 struct mv_xor_desc
*hw_desc
;
266 dev_dbg(mv_chan_to_devp(mv_chan
), "%s %d\n", __func__
, __LINE__
);
267 dev_dbg(mv_chan_to_devp(mv_chan
), "current_desc %x\n", current_desc
);
268 mv_chan_clean_completed_slots(mv_chan
);
270 /* free completed slots from the chain starting with
271 * the oldest descriptor
274 list_for_each_entry_safe(iter
, _iter
, &mv_chan
->chain
,
277 /* clean finished descriptors */
278 hw_desc
= iter
->hw_desc
;
279 if (hw_desc
->status
& XOR_DESC_SUCCESS
) {
280 cookie
= mv_desc_run_tx_complete_actions(iter
, mv_chan
,
283 /* done processing desc, clean slot */
284 mv_desc_clean_slot(iter
, mv_chan
);
286 /* break if we did cleaned the current */
287 if (iter
->async_tx
.phys
== current_desc
) {
292 if (iter
->async_tx
.phys
== current_desc
) {
299 if ((busy
== 0) && !list_empty(&mv_chan
->chain
)) {
300 if (current_cleaned
) {
302 * current descriptor cleaned and removed, run
305 iter
= list_entry(mv_chan
->chain
.next
,
306 struct mv_xor_desc_slot
,
308 mv_chan_start_new_chain(mv_chan
, iter
);
310 if (!list_is_last(&iter
->node
, &mv_chan
->chain
)) {
312 * descriptors are still waiting after
313 * current, trigger them
315 iter
= list_entry(iter
->node
.next
,
316 struct mv_xor_desc_slot
,
318 mv_chan_start_new_chain(mv_chan
, iter
);
321 * some descriptors are still waiting
324 tasklet_schedule(&mv_chan
->irq_tasklet
);
330 mv_chan
->dmachan
.completed_cookie
= cookie
;
333 static void mv_xor_tasklet(unsigned long data
)
335 struct mv_xor_chan
*chan
= (struct mv_xor_chan
*) data
;
337 spin_lock_bh(&chan
->lock
);
338 mv_chan_slot_cleanup(chan
);
339 spin_unlock_bh(&chan
->lock
);
342 static struct mv_xor_desc_slot
*
343 mv_chan_alloc_slot(struct mv_xor_chan
*mv_chan
)
345 struct mv_xor_desc_slot
*iter
;
347 spin_lock_bh(&mv_chan
->lock
);
349 if (!list_empty(&mv_chan
->free_slots
)) {
350 iter
= list_first_entry(&mv_chan
->free_slots
,
351 struct mv_xor_desc_slot
,
354 list_move_tail(&iter
->node
, &mv_chan
->allocated_slots
);
356 spin_unlock_bh(&mv_chan
->lock
);
358 /* pre-ack descriptor */
359 async_tx_ack(&iter
->async_tx
);
360 iter
->async_tx
.cookie
= -EBUSY
;
366 spin_unlock_bh(&mv_chan
->lock
);
368 /* try to free some slots if the allocation fails */
369 tasklet_schedule(&mv_chan
->irq_tasklet
);
374 /************************ DMA engine API functions ****************************/
376 mv_xor_tx_submit(struct dma_async_tx_descriptor
*tx
)
378 struct mv_xor_desc_slot
*sw_desc
= to_mv_xor_slot(tx
);
379 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(tx
->chan
);
380 struct mv_xor_desc_slot
*old_chain_tail
;
382 int new_hw_chain
= 1;
384 dev_dbg(mv_chan_to_devp(mv_chan
),
385 "%s sw_desc %p: async_tx %p\n",
386 __func__
, sw_desc
, &sw_desc
->async_tx
);
388 spin_lock_bh(&mv_chan
->lock
);
389 cookie
= dma_cookie_assign(tx
);
391 if (list_empty(&mv_chan
->chain
))
392 list_move_tail(&sw_desc
->node
, &mv_chan
->chain
);
396 old_chain_tail
= list_entry(mv_chan
->chain
.prev
,
397 struct mv_xor_desc_slot
,
399 list_move_tail(&sw_desc
->node
, &mv_chan
->chain
);
401 dev_dbg(mv_chan_to_devp(mv_chan
), "Append to last desc %pa\n",
402 &old_chain_tail
->async_tx
.phys
);
404 /* fix up the hardware chain */
405 mv_desc_set_next_desc(old_chain_tail
, sw_desc
->async_tx
.phys
);
407 /* if the channel is not busy */
408 if (!mv_chan_is_busy(mv_chan
)) {
409 u32 current_desc
= mv_chan_get_current_desc(mv_chan
);
411 * and the curren desc is the end of the chain before
412 * the append, then we need to start the channel
414 if (current_desc
== old_chain_tail
->async_tx
.phys
)
420 mv_chan_start_new_chain(mv_chan
, sw_desc
);
422 spin_unlock_bh(&mv_chan
->lock
);
427 /* returns the number of allocated descriptors */
428 static int mv_xor_alloc_chan_resources(struct dma_chan
*chan
)
433 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
434 struct mv_xor_desc_slot
*slot
= NULL
;
435 int num_descs_in_pool
= MV_XOR_POOL_SIZE
/MV_XOR_SLOT_SIZE
;
437 /* Allocate descriptor slots */
438 idx
= mv_chan
->slots_allocated
;
439 while (idx
< num_descs_in_pool
) {
440 slot
= kzalloc(sizeof(*slot
), GFP_KERNEL
);
442 dev_info(mv_chan_to_devp(mv_chan
),
443 "channel only initialized %d descriptor slots",
447 virt_desc
= mv_chan
->dma_desc_pool_virt
;
448 slot
->hw_desc
= virt_desc
+ idx
* MV_XOR_SLOT_SIZE
;
450 dma_async_tx_descriptor_init(&slot
->async_tx
, chan
);
451 slot
->async_tx
.tx_submit
= mv_xor_tx_submit
;
452 INIT_LIST_HEAD(&slot
->node
);
453 dma_desc
= mv_chan
->dma_desc_pool
;
454 slot
->async_tx
.phys
= dma_desc
+ idx
* MV_XOR_SLOT_SIZE
;
457 spin_lock_bh(&mv_chan
->lock
);
458 mv_chan
->slots_allocated
= idx
;
459 list_add_tail(&slot
->node
, &mv_chan
->free_slots
);
460 spin_unlock_bh(&mv_chan
->lock
);
463 dev_dbg(mv_chan_to_devp(mv_chan
),
464 "allocated %d descriptor slots\n",
465 mv_chan
->slots_allocated
);
467 return mv_chan
->slots_allocated
? : -ENOMEM
;
470 static struct dma_async_tx_descriptor
*
471 mv_xor_prep_dma_xor(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t
*src
,
472 unsigned int src_cnt
, size_t len
, unsigned long flags
)
474 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
475 struct mv_xor_desc_slot
*sw_desc
;
477 if (unlikely(len
< MV_XOR_MIN_BYTE_COUNT
))
480 BUG_ON(len
> MV_XOR_MAX_BYTE_COUNT
);
482 dev_dbg(mv_chan_to_devp(mv_chan
),
483 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
484 __func__
, src_cnt
, len
, &dest
, flags
);
486 sw_desc
= mv_chan_alloc_slot(mv_chan
);
488 sw_desc
->type
= DMA_XOR
;
489 sw_desc
->async_tx
.flags
= flags
;
490 mv_desc_init(sw_desc
, dest
, len
, flags
);
491 if (mv_chan
->op_in_desc
== XOR_MODE_IN_DESC
)
492 mv_desc_set_mode(sw_desc
);
494 mv_desc_set_src_addr(sw_desc
, src_cnt
, src
[src_cnt
]);
497 dev_dbg(mv_chan_to_devp(mv_chan
),
498 "%s sw_desc %p async_tx %p \n",
499 __func__
, sw_desc
, &sw_desc
->async_tx
);
500 return sw_desc
? &sw_desc
->async_tx
: NULL
;
503 static struct dma_async_tx_descriptor
*
504 mv_xor_prep_dma_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
505 size_t len
, unsigned long flags
)
508 * A MEMCPY operation is identical to an XOR operation with only
509 * a single source address.
511 return mv_xor_prep_dma_xor(chan
, dest
, &src
, 1, len
, flags
);
514 static struct dma_async_tx_descriptor
*
515 mv_xor_prep_dma_interrupt(struct dma_chan
*chan
, unsigned long flags
)
517 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
518 dma_addr_t src
, dest
;
521 src
= mv_chan
->dummy_src_addr
;
522 dest
= mv_chan
->dummy_dst_addr
;
523 len
= MV_XOR_MIN_BYTE_COUNT
;
526 * We implement the DMA_INTERRUPT operation as a minimum sized
527 * XOR operation with a single dummy source address.
529 return mv_xor_prep_dma_xor(chan
, dest
, &src
, 1, len
, flags
);
532 static void mv_xor_free_chan_resources(struct dma_chan
*chan
)
534 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
535 struct mv_xor_desc_slot
*iter
, *_iter
;
536 int in_use_descs
= 0;
538 spin_lock_bh(&mv_chan
->lock
);
540 mv_chan_slot_cleanup(mv_chan
);
542 list_for_each_entry_safe(iter
, _iter
, &mv_chan
->chain
,
545 list_move_tail(&iter
->node
, &mv_chan
->free_slots
);
547 list_for_each_entry_safe(iter
, _iter
, &mv_chan
->completed_slots
,
550 list_move_tail(&iter
->node
, &mv_chan
->free_slots
);
552 list_for_each_entry_safe(iter
, _iter
, &mv_chan
->allocated_slots
,
555 list_move_tail(&iter
->node
, &mv_chan
->free_slots
);
557 list_for_each_entry_safe_reverse(
558 iter
, _iter
, &mv_chan
->free_slots
, node
) {
559 list_del(&iter
->node
);
561 mv_chan
->slots_allocated
--;
564 dev_dbg(mv_chan_to_devp(mv_chan
), "%s slots_allocated %d\n",
565 __func__
, mv_chan
->slots_allocated
);
566 spin_unlock_bh(&mv_chan
->lock
);
569 dev_err(mv_chan_to_devp(mv_chan
),
570 "freeing %d in use descriptors!\n", in_use_descs
);
574 * mv_xor_status - poll the status of an XOR transaction
575 * @chan: XOR channel handle
576 * @cookie: XOR transaction identifier
577 * @txstate: XOR transactions state holder (or NULL)
579 static enum dma_status
mv_xor_status(struct dma_chan
*chan
,
581 struct dma_tx_state
*txstate
)
583 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
586 ret
= dma_cookie_status(chan
, cookie
, txstate
);
587 if (ret
== DMA_COMPLETE
)
590 spin_lock_bh(&mv_chan
->lock
);
591 mv_chan_slot_cleanup(mv_chan
);
592 spin_unlock_bh(&mv_chan
->lock
);
594 return dma_cookie_status(chan
, cookie
, txstate
);
597 static void mv_chan_dump_regs(struct mv_xor_chan
*chan
)
601 val
= readl_relaxed(XOR_CONFIG(chan
));
602 dev_err(mv_chan_to_devp(chan
), "config 0x%08x\n", val
);
604 val
= readl_relaxed(XOR_ACTIVATION(chan
));
605 dev_err(mv_chan_to_devp(chan
), "activation 0x%08x\n", val
);
607 val
= readl_relaxed(XOR_INTR_CAUSE(chan
));
608 dev_err(mv_chan_to_devp(chan
), "intr cause 0x%08x\n", val
);
610 val
= readl_relaxed(XOR_INTR_MASK(chan
));
611 dev_err(mv_chan_to_devp(chan
), "intr mask 0x%08x\n", val
);
613 val
= readl_relaxed(XOR_ERROR_CAUSE(chan
));
614 dev_err(mv_chan_to_devp(chan
), "error cause 0x%08x\n", val
);
616 val
= readl_relaxed(XOR_ERROR_ADDR(chan
));
617 dev_err(mv_chan_to_devp(chan
), "error addr 0x%08x\n", val
);
620 static void mv_chan_err_interrupt_handler(struct mv_xor_chan
*chan
,
623 if (intr_cause
& XOR_INT_ERR_DECODE
) {
624 dev_dbg(mv_chan_to_devp(chan
), "ignoring address decode error\n");
628 dev_err(mv_chan_to_devp(chan
), "error on chan %d. intr cause 0x%08x\n",
629 chan
->idx
, intr_cause
);
631 mv_chan_dump_regs(chan
);
635 static irqreturn_t
mv_xor_interrupt_handler(int irq
, void *data
)
637 struct mv_xor_chan
*chan
= data
;
638 u32 intr_cause
= mv_chan_get_intr_cause(chan
);
640 dev_dbg(mv_chan_to_devp(chan
), "intr cause %x\n", intr_cause
);
642 if (intr_cause
& XOR_INTR_ERRORS
)
643 mv_chan_err_interrupt_handler(chan
, intr_cause
);
645 tasklet_schedule(&chan
->irq_tasklet
);
647 mv_chan_clear_eoc_cause(chan
);
652 static void mv_xor_issue_pending(struct dma_chan
*chan
)
654 struct mv_xor_chan
*mv_chan
= to_mv_xor_chan(chan
);
656 if (mv_chan
->pending
>= MV_XOR_THRESHOLD
) {
657 mv_chan
->pending
= 0;
658 mv_chan_activate(mv_chan
);
663 * Perform a transaction to verify the HW works.
666 static int mv_chan_memcpy_self_test(struct mv_xor_chan
*mv_chan
)
670 dma_addr_t src_dma
, dest_dma
;
671 struct dma_chan
*dma_chan
;
673 struct dma_async_tx_descriptor
*tx
;
674 struct dmaengine_unmap_data
*unmap
;
677 src
= kmalloc(sizeof(u8
) * PAGE_SIZE
, GFP_KERNEL
);
681 dest
= kzalloc(sizeof(u8
) * PAGE_SIZE
, GFP_KERNEL
);
687 /* Fill in src buffer */
688 for (i
= 0; i
< PAGE_SIZE
; i
++)
689 ((u8
*) src
)[i
] = (u8
)i
;
691 dma_chan
= &mv_chan
->dmachan
;
692 if (mv_xor_alloc_chan_resources(dma_chan
) < 1) {
697 unmap
= dmaengine_get_unmap_data(dma_chan
->device
->dev
, 2, GFP_KERNEL
);
703 src_dma
= dma_map_page(dma_chan
->device
->dev
, virt_to_page(src
),
704 (size_t)src
& ~PAGE_MASK
, PAGE_SIZE
,
706 unmap
->addr
[0] = src_dma
;
708 ret
= dma_mapping_error(dma_chan
->device
->dev
, src_dma
);
715 dest_dma
= dma_map_page(dma_chan
->device
->dev
, virt_to_page(dest
),
716 (size_t)dest
& ~PAGE_MASK
, PAGE_SIZE
,
718 unmap
->addr
[1] = dest_dma
;
720 ret
= dma_mapping_error(dma_chan
->device
->dev
, dest_dma
);
726 unmap
->len
= PAGE_SIZE
;
728 tx
= mv_xor_prep_dma_memcpy(dma_chan
, dest_dma
, src_dma
,
731 dev_err(dma_chan
->device
->dev
,
732 "Self-test cannot prepare operation, disabling\n");
737 cookie
= mv_xor_tx_submit(tx
);
738 if (dma_submit_error(cookie
)) {
739 dev_err(dma_chan
->device
->dev
,
740 "Self-test submit error, disabling\n");
745 mv_xor_issue_pending(dma_chan
);
749 if (mv_xor_status(dma_chan
, cookie
, NULL
) !=
751 dev_err(dma_chan
->device
->dev
,
752 "Self-test copy timed out, disabling\n");
757 dma_sync_single_for_cpu(dma_chan
->device
->dev
, dest_dma
,
758 PAGE_SIZE
, DMA_FROM_DEVICE
);
759 if (memcmp(src
, dest
, PAGE_SIZE
)) {
760 dev_err(dma_chan
->device
->dev
,
761 "Self-test copy failed compare, disabling\n");
767 dmaengine_unmap_put(unmap
);
768 mv_xor_free_chan_resources(dma_chan
);
775 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
777 mv_chan_xor_self_test(struct mv_xor_chan
*mv_chan
)
781 struct page
*xor_srcs
[MV_XOR_NUM_SRC_TEST
];
782 dma_addr_t dma_srcs
[MV_XOR_NUM_SRC_TEST
];
784 struct dma_async_tx_descriptor
*tx
;
785 struct dmaengine_unmap_data
*unmap
;
786 struct dma_chan
*dma_chan
;
791 int src_count
= MV_XOR_NUM_SRC_TEST
;
793 for (src_idx
= 0; src_idx
< src_count
; src_idx
++) {
794 xor_srcs
[src_idx
] = alloc_page(GFP_KERNEL
);
795 if (!xor_srcs
[src_idx
]) {
797 __free_page(xor_srcs
[src_idx
]);
802 dest
= alloc_page(GFP_KERNEL
);
805 __free_page(xor_srcs
[src_idx
]);
809 /* Fill in src buffers */
810 for (src_idx
= 0; src_idx
< src_count
; src_idx
++) {
811 u8
*ptr
= page_address(xor_srcs
[src_idx
]);
812 for (i
= 0; i
< PAGE_SIZE
; i
++)
813 ptr
[i
] = (1 << src_idx
);
816 for (src_idx
= 0; src_idx
< src_count
; src_idx
++)
817 cmp_byte
^= (u8
) (1 << src_idx
);
819 cmp_word
= (cmp_byte
<< 24) | (cmp_byte
<< 16) |
820 (cmp_byte
<< 8) | cmp_byte
;
822 memset(page_address(dest
), 0, PAGE_SIZE
);
824 dma_chan
= &mv_chan
->dmachan
;
825 if (mv_xor_alloc_chan_resources(dma_chan
) < 1) {
830 unmap
= dmaengine_get_unmap_data(dma_chan
->device
->dev
, src_count
+ 1,
838 for (i
= 0; i
< src_count
; i
++) {
839 unmap
->addr
[i
] = dma_map_page(dma_chan
->device
->dev
, xor_srcs
[i
],
840 0, PAGE_SIZE
, DMA_TO_DEVICE
);
841 dma_srcs
[i
] = unmap
->addr
[i
];
842 ret
= dma_mapping_error(dma_chan
->device
->dev
, unmap
->addr
[i
]);
850 unmap
->addr
[src_count
] = dma_map_page(dma_chan
->device
->dev
, dest
, 0, PAGE_SIZE
,
852 dest_dma
= unmap
->addr
[src_count
];
853 ret
= dma_mapping_error(dma_chan
->device
->dev
, unmap
->addr
[src_count
]);
859 unmap
->len
= PAGE_SIZE
;
861 tx
= mv_xor_prep_dma_xor(dma_chan
, dest_dma
, dma_srcs
,
862 src_count
, PAGE_SIZE
, 0);
864 dev_err(dma_chan
->device
->dev
,
865 "Self-test cannot prepare operation, disabling\n");
870 cookie
= mv_xor_tx_submit(tx
);
871 if (dma_submit_error(cookie
)) {
872 dev_err(dma_chan
->device
->dev
,
873 "Self-test submit error, disabling\n");
878 mv_xor_issue_pending(dma_chan
);
882 if (mv_xor_status(dma_chan
, cookie
, NULL
) !=
884 dev_err(dma_chan
->device
->dev
,
885 "Self-test xor timed out, disabling\n");
890 dma_sync_single_for_cpu(dma_chan
->device
->dev
, dest_dma
,
891 PAGE_SIZE
, DMA_FROM_DEVICE
);
892 for (i
= 0; i
< (PAGE_SIZE
/ sizeof(u32
)); i
++) {
893 u32
*ptr
= page_address(dest
);
894 if (ptr
[i
] != cmp_word
) {
895 dev_err(dma_chan
->device
->dev
,
896 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
897 i
, ptr
[i
], cmp_word
);
904 dmaengine_unmap_put(unmap
);
905 mv_xor_free_chan_resources(dma_chan
);
909 __free_page(xor_srcs
[src_idx
]);
914 static int mv_xor_channel_remove(struct mv_xor_chan
*mv_chan
)
916 struct dma_chan
*chan
, *_chan
;
917 struct device
*dev
= mv_chan
->dmadev
.dev
;
919 dma_async_device_unregister(&mv_chan
->dmadev
);
921 dma_free_coherent(dev
, MV_XOR_POOL_SIZE
,
922 mv_chan
->dma_desc_pool_virt
, mv_chan
->dma_desc_pool
);
923 dma_unmap_single(dev
, mv_chan
->dummy_src_addr
,
924 MV_XOR_MIN_BYTE_COUNT
, DMA_FROM_DEVICE
);
925 dma_unmap_single(dev
, mv_chan
->dummy_dst_addr
,
926 MV_XOR_MIN_BYTE_COUNT
, DMA_TO_DEVICE
);
928 list_for_each_entry_safe(chan
, _chan
, &mv_chan
->dmadev
.channels
,
930 list_del(&chan
->device_node
);
933 free_irq(mv_chan
->irq
, mv_chan
);
938 static struct mv_xor_chan
*
939 mv_xor_channel_add(struct mv_xor_device
*xordev
,
940 struct platform_device
*pdev
,
941 int idx
, dma_cap_mask_t cap_mask
, int irq
)
944 struct mv_xor_chan
*mv_chan
;
945 struct dma_device
*dma_dev
;
947 mv_chan
= devm_kzalloc(&pdev
->dev
, sizeof(*mv_chan
), GFP_KERNEL
);
949 return ERR_PTR(-ENOMEM
);
953 if (xordev
->xor_type
== XOR_ORION
)
954 mv_chan
->op_in_desc
= XOR_MODE_IN_REG
;
956 mv_chan
->op_in_desc
= XOR_MODE_IN_DESC
;
958 dma_dev
= &mv_chan
->dmadev
;
961 * These source and destination dummy buffers are used to implement
962 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
963 * Hence, we only need to map the buffers at initialization-time.
965 mv_chan
->dummy_src_addr
= dma_map_single(dma_dev
->dev
,
966 mv_chan
->dummy_src
, MV_XOR_MIN_BYTE_COUNT
, DMA_FROM_DEVICE
);
967 mv_chan
->dummy_dst_addr
= dma_map_single(dma_dev
->dev
,
968 mv_chan
->dummy_dst
, MV_XOR_MIN_BYTE_COUNT
, DMA_TO_DEVICE
);
970 /* allocate coherent memory for hardware descriptors
971 * note: writecombine gives slightly better performance, but
972 * requires that we explicitly flush the writes
974 mv_chan
->dma_desc_pool_virt
=
975 dma_alloc_wc(&pdev
->dev
, MV_XOR_POOL_SIZE
, &mv_chan
->dma_desc_pool
,
977 if (!mv_chan
->dma_desc_pool_virt
)
978 return ERR_PTR(-ENOMEM
);
980 /* discover transaction capabilites from the platform data */
981 dma_dev
->cap_mask
= cap_mask
;
983 INIT_LIST_HEAD(&dma_dev
->channels
);
985 /* set base routines */
986 dma_dev
->device_alloc_chan_resources
= mv_xor_alloc_chan_resources
;
987 dma_dev
->device_free_chan_resources
= mv_xor_free_chan_resources
;
988 dma_dev
->device_tx_status
= mv_xor_status
;
989 dma_dev
->device_issue_pending
= mv_xor_issue_pending
;
990 dma_dev
->dev
= &pdev
->dev
;
992 /* set prep routines based on capability */
993 if (dma_has_cap(DMA_INTERRUPT
, dma_dev
->cap_mask
))
994 dma_dev
->device_prep_dma_interrupt
= mv_xor_prep_dma_interrupt
;
995 if (dma_has_cap(DMA_MEMCPY
, dma_dev
->cap_mask
))
996 dma_dev
->device_prep_dma_memcpy
= mv_xor_prep_dma_memcpy
;
997 if (dma_has_cap(DMA_XOR
, dma_dev
->cap_mask
)) {
998 dma_dev
->max_xor
= 8;
999 dma_dev
->device_prep_dma_xor
= mv_xor_prep_dma_xor
;
1002 mv_chan
->mmr_base
= xordev
->xor_base
;
1003 mv_chan
->mmr_high_base
= xordev
->xor_high_base
;
1004 tasklet_init(&mv_chan
->irq_tasklet
, mv_xor_tasklet
, (unsigned long)
1007 /* clear errors before enabling interrupts */
1008 mv_chan_clear_err_status(mv_chan
);
1010 ret
= request_irq(mv_chan
->irq
, mv_xor_interrupt_handler
,
1011 0, dev_name(&pdev
->dev
), mv_chan
);
1015 mv_chan_unmask_interrupts(mv_chan
);
1017 if (mv_chan
->op_in_desc
== XOR_MODE_IN_DESC
)
1018 mv_chan_set_mode(mv_chan
, XOR_OPERATION_MODE_IN_DESC
);
1020 mv_chan_set_mode(mv_chan
, XOR_OPERATION_MODE_XOR
);
1022 spin_lock_init(&mv_chan
->lock
);
1023 INIT_LIST_HEAD(&mv_chan
->chain
);
1024 INIT_LIST_HEAD(&mv_chan
->completed_slots
);
1025 INIT_LIST_HEAD(&mv_chan
->free_slots
);
1026 INIT_LIST_HEAD(&mv_chan
->allocated_slots
);
1027 mv_chan
->dmachan
.device
= dma_dev
;
1028 dma_cookie_init(&mv_chan
->dmachan
);
1030 list_add_tail(&mv_chan
->dmachan
.device_node
, &dma_dev
->channels
);
1032 if (dma_has_cap(DMA_MEMCPY
, dma_dev
->cap_mask
)) {
1033 ret
= mv_chan_memcpy_self_test(mv_chan
);
1034 dev_dbg(&pdev
->dev
, "memcpy self test returned %d\n", ret
);
1039 if (dma_has_cap(DMA_XOR
, dma_dev
->cap_mask
)) {
1040 ret
= mv_chan_xor_self_test(mv_chan
);
1041 dev_dbg(&pdev
->dev
, "xor self test returned %d\n", ret
);
1046 dev_info(&pdev
->dev
, "Marvell XOR (%s): ( %s%s%s)\n",
1047 mv_chan
->op_in_desc
? "Descriptor Mode" : "Registers Mode",
1048 dma_has_cap(DMA_XOR
, dma_dev
->cap_mask
) ? "xor " : "",
1049 dma_has_cap(DMA_MEMCPY
, dma_dev
->cap_mask
) ? "cpy " : "",
1050 dma_has_cap(DMA_INTERRUPT
, dma_dev
->cap_mask
) ? "intr " : "");
1052 dma_async_device_register(dma_dev
);
1056 free_irq(mv_chan
->irq
, mv_chan
);
1058 dma_free_coherent(&pdev
->dev
, MV_XOR_POOL_SIZE
,
1059 mv_chan
->dma_desc_pool_virt
, mv_chan
->dma_desc_pool
);
1060 return ERR_PTR(ret
);
1064 mv_xor_conf_mbus_windows(struct mv_xor_device
*xordev
,
1065 const struct mbus_dram_target_info
*dram
)
1067 void __iomem
*base
= xordev
->xor_high_base
;
1071 for (i
= 0; i
< 8; i
++) {
1072 writel(0, base
+ WINDOW_BASE(i
));
1073 writel(0, base
+ WINDOW_SIZE(i
));
1075 writel(0, base
+ WINDOW_REMAP_HIGH(i
));
1078 for (i
= 0; i
< dram
->num_cs
; i
++) {
1079 const struct mbus_dram_window
*cs
= dram
->cs
+ i
;
1081 writel((cs
->base
& 0xffff0000) |
1082 (cs
->mbus_attr
<< 8) |
1083 dram
->mbus_dram_target_id
, base
+ WINDOW_BASE(i
));
1084 writel((cs
->size
- 1) & 0xffff0000, base
+ WINDOW_SIZE(i
));
1086 win_enable
|= (1 << i
);
1087 win_enable
|= 3 << (16 + (2 * i
));
1090 writel(win_enable
, base
+ WINDOW_BAR_ENABLE(0));
1091 writel(win_enable
, base
+ WINDOW_BAR_ENABLE(1));
1092 writel(0, base
+ WINDOW_OVERRIDE_CTRL(0));
1093 writel(0, base
+ WINDOW_OVERRIDE_CTRL(1));
1097 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device
*xordev
)
1099 void __iomem
*base
= xordev
->xor_high_base
;
1103 for (i
= 0; i
< 8; i
++) {
1104 writel(0, base
+ WINDOW_BASE(i
));
1105 writel(0, base
+ WINDOW_SIZE(i
));
1107 writel(0, base
+ WINDOW_REMAP_HIGH(i
));
1110 * For Armada3700 open default 4GB Mbus window. The dram
1111 * related configuration are done at AXIS level.
1113 writel(0xffff0000, base
+ WINDOW_SIZE(0));
1115 win_enable
|= 3 << 16;
1117 writel(win_enable
, base
+ WINDOW_BAR_ENABLE(0));
1118 writel(win_enable
, base
+ WINDOW_BAR_ENABLE(1));
1119 writel(0, base
+ WINDOW_OVERRIDE_CTRL(0));
1120 writel(0, base
+ WINDOW_OVERRIDE_CTRL(1));
1124 * Since this XOR driver is basically used only for RAID5, we don't
1125 * need to care about synchronizing ->suspend with DMA activity,
1126 * because the DMA engine will naturally be quiet due to the block
1127 * devices being suspended.
1129 static int mv_xor_suspend(struct platform_device
*pdev
, pm_message_t state
)
1131 struct mv_xor_device
*xordev
= platform_get_drvdata(pdev
);
1134 for (i
= 0; i
< MV_XOR_MAX_CHANNELS
; i
++) {
1135 struct mv_xor_chan
*mv_chan
= xordev
->channels
[i
];
1140 mv_chan
->saved_config_reg
=
1141 readl_relaxed(XOR_CONFIG(mv_chan
));
1142 mv_chan
->saved_int_mask_reg
=
1143 readl_relaxed(XOR_INTR_MASK(mv_chan
));
1149 static int mv_xor_resume(struct platform_device
*dev
)
1151 struct mv_xor_device
*xordev
= platform_get_drvdata(dev
);
1152 const struct mbus_dram_target_info
*dram
;
1155 for (i
= 0; i
< MV_XOR_MAX_CHANNELS
; i
++) {
1156 struct mv_xor_chan
*mv_chan
= xordev
->channels
[i
];
1161 writel_relaxed(mv_chan
->saved_config_reg
,
1162 XOR_CONFIG(mv_chan
));
1163 writel_relaxed(mv_chan
->saved_int_mask_reg
,
1164 XOR_INTR_MASK(mv_chan
));
1167 if (xordev
->xor_type
== XOR_ARMADA_37XX
) {
1168 mv_xor_conf_mbus_windows_a3700(xordev
);
1172 dram
= mv_mbus_dram_info();
1174 mv_xor_conf_mbus_windows(xordev
, dram
);
1179 static const struct of_device_id mv_xor_dt_ids
[] = {
1180 { .compatible
= "marvell,orion-xor", .data
= (void *)XOR_ORION
},
1181 { .compatible
= "marvell,armada-380-xor", .data
= (void *)XOR_ARMADA_38X
},
1182 { .compatible
= "marvell,armada-3700-xor", .data
= (void *)XOR_ARMADA_37XX
},
1186 static unsigned int mv_xor_engine_count
;
1188 static int mv_xor_probe(struct platform_device
*pdev
)
1190 const struct mbus_dram_target_info
*dram
;
1191 struct mv_xor_device
*xordev
;
1192 struct mv_xor_platform_data
*pdata
= dev_get_platdata(&pdev
->dev
);
1193 struct resource
*res
;
1194 unsigned int max_engines
, max_channels
;
1197 dev_notice(&pdev
->dev
, "Marvell shared XOR driver\n");
1199 xordev
= devm_kzalloc(&pdev
->dev
, sizeof(*xordev
), GFP_KERNEL
);
1203 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1207 xordev
->xor_base
= devm_ioremap(&pdev
->dev
, res
->start
,
1208 resource_size(res
));
1209 if (!xordev
->xor_base
)
1212 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1216 xordev
->xor_high_base
= devm_ioremap(&pdev
->dev
, res
->start
,
1217 resource_size(res
));
1218 if (!xordev
->xor_high_base
)
1221 platform_set_drvdata(pdev
, xordev
);
1225 * We need to know which type of XOR device we use before
1226 * setting up. In non-dt case it can only be the legacy one.
1228 xordev
->xor_type
= XOR_ORION
;
1229 if (pdev
->dev
.of_node
) {
1230 const struct of_device_id
*of_id
=
1231 of_match_device(mv_xor_dt_ids
,
1234 xordev
->xor_type
= (uintptr_t)of_id
->data
;
1238 * (Re-)program MBUS remapping windows if we are asked to.
1240 if (xordev
->xor_type
== XOR_ARMADA_37XX
) {
1241 mv_xor_conf_mbus_windows_a3700(xordev
);
1243 dram
= mv_mbus_dram_info();
1245 mv_xor_conf_mbus_windows(xordev
, dram
);
1248 /* Not all platforms can gate the clock, so it is not
1249 * an error if the clock does not exists.
1251 xordev
->clk
= clk_get(&pdev
->dev
, NULL
);
1252 if (!IS_ERR(xordev
->clk
))
1253 clk_prepare_enable(xordev
->clk
);
1256 * We don't want to have more than one channel per CPU in
1257 * order for async_tx to perform well. So we limit the number
1258 * of engines and channels so that we take into account this
1259 * constraint. Note that we also want to use channels from
1260 * separate engines when possible. For dual-CPU Armada 3700
1261 * SoC with single XOR engine allow using its both channels.
1263 max_engines
= num_present_cpus();
1264 if (xordev
->xor_type
== XOR_ARMADA_37XX
)
1265 max_channels
= num_present_cpus();
1267 max_channels
= min_t(unsigned int,
1268 MV_XOR_MAX_CHANNELS
,
1269 DIV_ROUND_UP(num_present_cpus(), 2));
1271 if (mv_xor_engine_count
>= max_engines
)
1274 if (pdev
->dev
.of_node
) {
1275 struct device_node
*np
;
1278 for_each_child_of_node(pdev
->dev
.of_node
, np
) {
1279 struct mv_xor_chan
*chan
;
1280 dma_cap_mask_t cap_mask
;
1283 if (i
>= max_channels
)
1286 dma_cap_zero(cap_mask
);
1287 dma_cap_set(DMA_MEMCPY
, cap_mask
);
1288 dma_cap_set(DMA_XOR
, cap_mask
);
1289 dma_cap_set(DMA_INTERRUPT
, cap_mask
);
1291 irq
= irq_of_parse_and_map(np
, 0);
1294 goto err_channel_add
;
1297 chan
= mv_xor_channel_add(xordev
, pdev
, i
,
1300 ret
= PTR_ERR(chan
);
1301 irq_dispose_mapping(irq
);
1302 goto err_channel_add
;
1305 xordev
->channels
[i
] = chan
;
1308 } else if (pdata
&& pdata
->channels
) {
1309 for (i
= 0; i
< max_channels
; i
++) {
1310 struct mv_xor_channel_data
*cd
;
1311 struct mv_xor_chan
*chan
;
1314 cd
= &pdata
->channels
[i
];
1317 goto err_channel_add
;
1320 irq
= platform_get_irq(pdev
, i
);
1323 goto err_channel_add
;
1326 chan
= mv_xor_channel_add(xordev
, pdev
, i
,
1329 ret
= PTR_ERR(chan
);
1330 goto err_channel_add
;
1333 xordev
->channels
[i
] = chan
;
1340 for (i
= 0; i
< MV_XOR_MAX_CHANNELS
; i
++)
1341 if (xordev
->channels
[i
]) {
1342 mv_xor_channel_remove(xordev
->channels
[i
]);
1343 if (pdev
->dev
.of_node
)
1344 irq_dispose_mapping(xordev
->channels
[i
]->irq
);
1347 if (!IS_ERR(xordev
->clk
)) {
1348 clk_disable_unprepare(xordev
->clk
);
1349 clk_put(xordev
->clk
);
1355 static struct platform_driver mv_xor_driver
= {
1356 .probe
= mv_xor_probe
,
1357 .suspend
= mv_xor_suspend
,
1358 .resume
= mv_xor_resume
,
1360 .name
= MV_XOR_NAME
,
1361 .of_match_table
= of_match_ptr(mv_xor_dt_ids
),
1366 static int __init
mv_xor_init(void)
1368 return platform_driver_register(&mv_xor_driver
);
1370 device_initcall(mv_xor_init
);
1373 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1374 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1375 MODULE_LICENSE("GPL");