2 * driver/dma/coh901318.c
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * DMA driver for COH 901 318
7 * Author: Per Friden <per.friden@stericsson.com>
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h> /* printk() */
13 #include <linux/fs.h> /* everything... */
14 #include <linux/scatterlist.h>
15 #include <linux/slab.h> /* kmalloc() */
16 #include <linux/dmaengine.h>
17 #include <linux/platform_device.h>
18 #include <linux/device.h>
19 #include <linux/irqreturn.h>
20 #include <linux/interrupt.h>
22 #include <linux/uaccess.h>
23 #include <linux/debugfs.h>
24 #include <mach/coh901318.h>
26 #include "coh901318_lli.h"
28 #define COHC_2_DEV(cohc) (&cohc->chan.dev->device)
31 #define COH_DBG(x) ({ if (1) x; 0; })
33 #define COH_DBG(x) ({ if (0) x; 0; })
36 struct coh901318_desc
{
37 struct dma_async_tx_descriptor desc
;
38 struct list_head node
;
39 struct scatterlist
*sg
;
41 struct coh901318_lli
*lli
;
42 enum dma_transfer_direction dir
;
48 struct coh901318_base
{
50 void __iomem
*virtbase
;
51 struct coh901318_pool pool
;
53 struct dma_device dma_slave
;
54 struct dma_device dma_memcpy
;
55 struct coh901318_chan
*chans
;
56 struct coh901318_platform
*platform
;
59 struct coh901318_chan
{
65 struct work_struct free_work
;
68 struct tasklet_struct tasklet
;
70 struct list_head active
;
71 struct list_head queue
;
72 struct list_head free
;
74 unsigned long nbr_active_done
;
80 struct coh901318_base
*base
;
83 static void coh901318_list_print(struct coh901318_chan
*cohc
,
84 struct coh901318_lli
*lli
)
86 struct coh901318_lli
*l
= lli
;
90 dev_vdbg(COHC_2_DEV(cohc
), "i %d, lli %p, ctrl 0x%x, src 0x%x"
91 ", dst 0x%x, link 0x%x virt_link_addr 0x%p\n",
92 i
, l
, l
->control
, l
->src_addr
, l
->dst_addr
,
93 l
->link_addr
, l
->virt_link_addr
);
95 l
= l
->virt_link_addr
;
99 #ifdef CONFIG_DEBUG_FS
101 #define COH901318_DEBUGFS_ASSIGN(x, y) (x = y)
103 static struct coh901318_base
*debugfs_dma_base
;
104 static struct dentry
*dma_dentry
;
106 static int coh901318_debugfs_open(struct inode
*inode
, struct file
*file
)
109 file
->private_data
= inode
->i_private
;
113 static int coh901318_debugfs_read(struct file
*file
, char __user
*buf
,
114 size_t count
, loff_t
*f_pos
)
116 u64 started_channels
= debugfs_dma_base
->pm
.started_channels
;
117 int pool_count
= debugfs_dma_base
->pool
.debugfs_pool_counter
;
124 dev_buf
= kmalloc(4*1024, GFP_KERNEL
);
129 tmp
+= sprintf(tmp
, "DMA -- enabled dma channels\n");
131 for (i
= 0; i
< debugfs_dma_base
->platform
->max_channels
; i
++)
132 if (started_channels
& (1 << i
))
133 tmp
+= sprintf(tmp
, "channel %d\n", i
);
135 tmp
+= sprintf(tmp
, "Pool alloc nbr %d\n", pool_count
);
136 dev_size
= tmp
- dev_buf
;
138 /* No more to read if offset != 0 */
139 if (*f_pos
> dev_size
)
142 if (count
> dev_size
- *f_pos
)
143 count
= dev_size
- *f_pos
;
145 if (copy_to_user(buf
, dev_buf
+ *f_pos
, count
))
158 static const struct file_operations coh901318_debugfs_status_operations
= {
159 .owner
= THIS_MODULE
,
160 .open
= coh901318_debugfs_open
,
161 .read
= coh901318_debugfs_read
,
162 .llseek
= default_llseek
,
166 static int __init
init_coh901318_debugfs(void)
169 dma_dentry
= debugfs_create_dir("dma", NULL
);
171 (void) debugfs_create_file("status",
174 &coh901318_debugfs_status_operations
);
178 static void __exit
exit_coh901318_debugfs(void)
180 debugfs_remove_recursive(dma_dentry
);
183 module_init(init_coh901318_debugfs
);
184 module_exit(exit_coh901318_debugfs
);
187 #define COH901318_DEBUGFS_ASSIGN(x, y)
189 #endif /* CONFIG_DEBUG_FS */
191 static inline struct coh901318_chan
*to_coh901318_chan(struct dma_chan
*chan
)
193 return container_of(chan
, struct coh901318_chan
, chan
);
196 static inline dma_addr_t
197 cohc_dev_addr(struct coh901318_chan
*cohc
)
199 /* Runtime supplied address will take precedence */
200 if (cohc
->runtime_addr
)
201 return cohc
->runtime_addr
;
202 return cohc
->base
->platform
->chan_conf
[cohc
->id
].dev_addr
;
205 static inline const struct coh901318_params
*
206 cohc_chan_param(struct coh901318_chan
*cohc
)
208 return &cohc
->base
->platform
->chan_conf
[cohc
->id
].param
;
211 static inline const struct coh_dma_channel
*
212 cohc_chan_conf(struct coh901318_chan
*cohc
)
214 return &cohc
->base
->platform
->chan_conf
[cohc
->id
];
217 static void enable_powersave(struct coh901318_chan
*cohc
)
220 struct powersave
*pm
= &cohc
->base
->pm
;
222 spin_lock_irqsave(&pm
->lock
, flags
);
224 pm
->started_channels
&= ~(1ULL << cohc
->id
);
226 if (!pm
->started_channels
) {
227 /* DMA no longer intends to access memory */
228 cohc
->base
->platform
->access_memory_state(cohc
->base
->dev
,
232 spin_unlock_irqrestore(&pm
->lock
, flags
);
234 static void disable_powersave(struct coh901318_chan
*cohc
)
237 struct powersave
*pm
= &cohc
->base
->pm
;
239 spin_lock_irqsave(&pm
->lock
, flags
);
241 if (!pm
->started_channels
) {
242 /* DMA intends to access memory */
243 cohc
->base
->platform
->access_memory_state(cohc
->base
->dev
,
247 pm
->started_channels
|= (1ULL << cohc
->id
);
249 spin_unlock_irqrestore(&pm
->lock
, flags
);
252 static inline int coh901318_set_ctrl(struct coh901318_chan
*cohc
, u32 control
)
254 int channel
= cohc
->id
;
255 void __iomem
*virtbase
= cohc
->base
->virtbase
;
258 virtbase
+ COH901318_CX_CTRL
+
259 COH901318_CX_CTRL_SPACING
* channel
);
263 static inline int coh901318_set_conf(struct coh901318_chan
*cohc
, u32 conf
)
265 int channel
= cohc
->id
;
266 void __iomem
*virtbase
= cohc
->base
->virtbase
;
269 virtbase
+ COH901318_CX_CFG
+
270 COH901318_CX_CFG_SPACING
*channel
);
275 static int coh901318_start(struct coh901318_chan
*cohc
)
278 int channel
= cohc
->id
;
279 void __iomem
*virtbase
= cohc
->base
->virtbase
;
281 disable_powersave(cohc
);
283 val
= readl(virtbase
+ COH901318_CX_CFG
+
284 COH901318_CX_CFG_SPACING
* channel
);
287 val
|= COH901318_CX_CFG_CH_ENABLE
;
288 writel(val
, virtbase
+ COH901318_CX_CFG
+
289 COH901318_CX_CFG_SPACING
* channel
);
294 static int coh901318_prep_linked_list(struct coh901318_chan
*cohc
,
295 struct coh901318_lli
*lli
)
297 int channel
= cohc
->id
;
298 void __iomem
*virtbase
= cohc
->base
->virtbase
;
300 BUG_ON(readl(virtbase
+ COH901318_CX_STAT
+
301 COH901318_CX_STAT_SPACING
*channel
) &
302 COH901318_CX_STAT_ACTIVE
);
304 writel(lli
->src_addr
,
305 virtbase
+ COH901318_CX_SRC_ADDR
+
306 COH901318_CX_SRC_ADDR_SPACING
* channel
);
308 writel(lli
->dst_addr
, virtbase
+
309 COH901318_CX_DST_ADDR
+
310 COH901318_CX_DST_ADDR_SPACING
* channel
);
312 writel(lli
->link_addr
, virtbase
+ COH901318_CX_LNK_ADDR
+
313 COH901318_CX_LNK_ADDR_SPACING
* channel
);
315 writel(lli
->control
, virtbase
+ COH901318_CX_CTRL
+
316 COH901318_CX_CTRL_SPACING
* channel
);
321 coh901318_assign_cookie(struct coh901318_chan
*cohc
,
322 struct coh901318_desc
*cohd
)
324 dma_cookie_t cookie
= cohc
->chan
.cookie
;
329 cohc
->chan
.cookie
= cookie
;
330 cohd
->desc
.cookie
= cookie
;
335 static struct coh901318_desc
*
336 coh901318_desc_get(struct coh901318_chan
*cohc
)
338 struct coh901318_desc
*desc
;
340 if (list_empty(&cohc
->free
)) {
341 /* alloc new desc because we're out of used ones
342 * TODO: alloc a pile of descs instead of just one,
343 * avoid many small allocations.
345 desc
= kzalloc(sizeof(struct coh901318_desc
), GFP_NOWAIT
);
348 INIT_LIST_HEAD(&desc
->node
);
349 dma_async_tx_descriptor_init(&desc
->desc
, &cohc
->chan
);
351 /* Reuse an old desc. */
352 desc
= list_first_entry(&cohc
->free
,
353 struct coh901318_desc
,
355 list_del(&desc
->node
);
356 /* Initialize it a bit so it's not insane */
359 desc
->desc
.callback
= NULL
;
360 desc
->desc
.callback_param
= NULL
;
368 coh901318_desc_free(struct coh901318_chan
*cohc
, struct coh901318_desc
*cohd
)
370 list_add_tail(&cohd
->node
, &cohc
->free
);
373 /* call with irq lock held */
375 coh901318_desc_submit(struct coh901318_chan
*cohc
, struct coh901318_desc
*desc
)
377 list_add_tail(&desc
->node
, &cohc
->active
);
380 static struct coh901318_desc
*
381 coh901318_first_active_get(struct coh901318_chan
*cohc
)
383 struct coh901318_desc
*d
;
385 if (list_empty(&cohc
->active
))
388 d
= list_first_entry(&cohc
->active
,
389 struct coh901318_desc
,
395 coh901318_desc_remove(struct coh901318_desc
*cohd
)
397 list_del(&cohd
->node
);
401 coh901318_desc_queue(struct coh901318_chan
*cohc
, struct coh901318_desc
*desc
)
403 list_add_tail(&desc
->node
, &cohc
->queue
);
406 static struct coh901318_desc
*
407 coh901318_first_queued(struct coh901318_chan
*cohc
)
409 struct coh901318_desc
*d
;
411 if (list_empty(&cohc
->queue
))
414 d
= list_first_entry(&cohc
->queue
,
415 struct coh901318_desc
,
420 static inline u32
coh901318_get_bytes_in_lli(struct coh901318_lli
*in_lli
)
422 struct coh901318_lli
*lli
= in_lli
;
426 bytes
+= lli
->control
& COH901318_CX_CTRL_TC_VALUE_MASK
;
427 lli
= lli
->virt_link_addr
;
433 * Get the number of bytes left to transfer on this channel,
434 * it is unwise to call this before stopping the channel for
435 * absolute measures, but for a rough guess you can still call
438 static u32
coh901318_get_bytes_left(struct dma_chan
*chan
)
440 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
441 struct coh901318_desc
*cohd
;
442 struct list_head
*pos
;
447 spin_lock_irqsave(&cohc
->lock
, flags
);
450 * If there are many queued jobs, we iterate and add the
451 * size of them all. We take a special look on the first
452 * job though, since it is probably active.
454 list_for_each(pos
, &cohc
->active
) {
456 * The first job in the list will be working on the
457 * hardware. The job can be stopped but still active,
458 * so that the transfer counter is somewhere inside
461 cohd
= list_entry(pos
, struct coh901318_desc
, node
);
464 struct coh901318_lli
*lli
;
467 /* Read current transfer count value */
468 left
= readl(cohc
->base
->virtbase
+
470 COH901318_CX_CTRL_SPACING
* cohc
->id
) &
471 COH901318_CX_CTRL_TC_VALUE_MASK
;
473 /* See if the transfer is linked... */
474 ladd
= readl(cohc
->base
->virtbase
+
475 COH901318_CX_LNK_ADDR
+
476 COH901318_CX_LNK_ADDR_SPACING
*
478 ~COH901318_CX_LNK_LINK_IMMEDIATE
;
479 /* Single transaction */
484 * Linked transaction, follow the lli, find the
485 * currently processing lli, and proceed to the next
488 while (lli
&& lli
->link_addr
!= ladd
)
489 lli
= lli
->virt_link_addr
;
492 lli
= lli
->virt_link_addr
;
495 * Follow remaining lli links around to count the total
496 * number of bytes left
498 left
+= coh901318_get_bytes_in_lli(lli
);
500 left
+= coh901318_get_bytes_in_lli(cohd
->lli
);
505 /* Also count bytes in the queued jobs */
506 list_for_each(pos
, &cohc
->queue
) {
507 cohd
= list_entry(pos
, struct coh901318_desc
, node
);
508 left
+= coh901318_get_bytes_in_lli(cohd
->lli
);
511 spin_unlock_irqrestore(&cohc
->lock
, flags
);
517 * Pauses a transfer without losing data. Enables power save.
518 * Use this function in conjunction with coh901318_resume.
520 static void coh901318_pause(struct dma_chan
*chan
)
524 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
525 int channel
= cohc
->id
;
526 void __iomem
*virtbase
= cohc
->base
->virtbase
;
528 spin_lock_irqsave(&cohc
->lock
, flags
);
530 /* Disable channel in HW */
531 val
= readl(virtbase
+ COH901318_CX_CFG
+
532 COH901318_CX_CFG_SPACING
* channel
);
534 /* Stopping infinite transfer */
535 if ((val
& COH901318_CX_CTRL_TC_ENABLE
) == 0 &&
536 (val
& COH901318_CX_CFG_CH_ENABLE
))
540 val
&= ~COH901318_CX_CFG_CH_ENABLE
;
541 /* Enable twice, HW bug work around */
542 writel(val
, virtbase
+ COH901318_CX_CFG
+
543 COH901318_CX_CFG_SPACING
* channel
);
544 writel(val
, virtbase
+ COH901318_CX_CFG
+
545 COH901318_CX_CFG_SPACING
* channel
);
547 /* Spin-wait for it to actually go inactive */
548 while (readl(virtbase
+ COH901318_CX_STAT
+COH901318_CX_STAT_SPACING
*
549 channel
) & COH901318_CX_STAT_ACTIVE
)
552 /* Check if we stopped an active job */
553 if ((readl(virtbase
+ COH901318_CX_CTRL
+COH901318_CX_CTRL_SPACING
*
554 channel
) & COH901318_CX_CTRL_TC_VALUE_MASK
) > 0)
557 enable_powersave(cohc
);
559 spin_unlock_irqrestore(&cohc
->lock
, flags
);
562 /* Resumes a transfer that has been stopped via 300_dma_stop(..).
563 Power save is handled.
565 static void coh901318_resume(struct dma_chan
*chan
)
569 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
570 int channel
= cohc
->id
;
572 spin_lock_irqsave(&cohc
->lock
, flags
);
574 disable_powersave(cohc
);
577 /* Enable channel in HW */
578 val
= readl(cohc
->base
->virtbase
+ COH901318_CX_CFG
+
579 COH901318_CX_CFG_SPACING
* channel
);
581 val
|= COH901318_CX_CFG_CH_ENABLE
;
583 writel(val
, cohc
->base
->virtbase
+ COH901318_CX_CFG
+
584 COH901318_CX_CFG_SPACING
*channel
);
589 spin_unlock_irqrestore(&cohc
->lock
, flags
);
592 bool coh901318_filter_id(struct dma_chan
*chan
, void *chan_id
)
594 unsigned int ch_nr
= (unsigned int) chan_id
;
596 if (ch_nr
== to_coh901318_chan(chan
)->id
)
601 EXPORT_SYMBOL(coh901318_filter_id
);
604 * DMA channel allocation
606 static int coh901318_config(struct coh901318_chan
*cohc
,
607 struct coh901318_params
*param
)
610 const struct coh901318_params
*p
;
611 int channel
= cohc
->id
;
612 void __iomem
*virtbase
= cohc
->base
->virtbase
;
614 spin_lock_irqsave(&cohc
->lock
, flags
);
619 p
= &cohc
->base
->platform
->chan_conf
[channel
].param
;
621 /* Clear any pending BE or TC interrupt */
623 writel(1 << channel
, virtbase
+ COH901318_BE_INT_CLEAR1
);
624 writel(1 << channel
, virtbase
+ COH901318_TC_INT_CLEAR1
);
626 writel(1 << (channel
- 32), virtbase
+
627 COH901318_BE_INT_CLEAR2
);
628 writel(1 << (channel
- 32), virtbase
+
629 COH901318_TC_INT_CLEAR2
);
632 coh901318_set_conf(cohc
, p
->config
);
633 coh901318_set_ctrl(cohc
, p
->ctrl_lli_last
);
635 spin_unlock_irqrestore(&cohc
->lock
, flags
);
640 /* must lock when calling this function
641 * start queued jobs, if any
642 * TODO: start all queued jobs in one go
644 * Returns descriptor if queued job is started otherwise NULL.
645 * If the queue is empty NULL is returned.
647 static struct coh901318_desc
*coh901318_queue_start(struct coh901318_chan
*cohc
)
649 struct coh901318_desc
*cohd
;
652 * start queued jobs, if any
653 * TODO: transmit all queued jobs in one go
655 cohd
= coh901318_first_queued(cohc
);
658 /* Remove from queue */
659 coh901318_desc_remove(cohd
);
660 /* initiate DMA job */
663 coh901318_desc_submit(cohc
, cohd
);
665 /* Program the transaction head */
666 coh901318_set_conf(cohc
, cohd
->head_config
);
667 coh901318_set_ctrl(cohc
, cohd
->head_ctrl
);
668 coh901318_prep_linked_list(cohc
, cohd
->lli
);
670 /* start dma job on this channel */
671 coh901318_start(cohc
);
679 * This tasklet is called from the interrupt handler to
680 * handle each descriptor (DMA job) that is sent to a channel.
682 static void dma_tasklet(unsigned long data
)
684 struct coh901318_chan
*cohc
= (struct coh901318_chan
*) data
;
685 struct coh901318_desc
*cohd_fin
;
687 dma_async_tx_callback callback
;
688 void *callback_param
;
690 dev_vdbg(COHC_2_DEV(cohc
), "[%s] chan_id %d"
691 " nbr_active_done %ld\n", __func__
,
692 cohc
->id
, cohc
->nbr_active_done
);
694 spin_lock_irqsave(&cohc
->lock
, flags
);
696 /* get first active descriptor entry from list */
697 cohd_fin
= coh901318_first_active_get(cohc
);
699 if (cohd_fin
== NULL
)
702 /* locate callback to client */
703 callback
= cohd_fin
->desc
.callback
;
704 callback_param
= cohd_fin
->desc
.callback_param
;
706 /* sign this job as completed on the channel */
707 cohc
->chan
.completed_cookie
= cohd_fin
->desc
.cookie
;
709 /* release the lli allocation and remove the descriptor */
710 coh901318_lli_free(&cohc
->base
->pool
, &cohd_fin
->lli
);
712 /* return desc to free-list */
713 coh901318_desc_remove(cohd_fin
);
714 coh901318_desc_free(cohc
, cohd_fin
);
716 spin_unlock_irqrestore(&cohc
->lock
, flags
);
718 /* Call the callback when we're done */
720 callback(callback_param
);
722 spin_lock_irqsave(&cohc
->lock
, flags
);
725 * If another interrupt fired while the tasklet was scheduling,
726 * we don't get called twice, so we have this number of active
727 * counter that keep track of the number of IRQs expected to
728 * be handled for this channel. If there happen to be more than
729 * one IRQ to be ack:ed, we simply schedule this tasklet again.
731 cohc
->nbr_active_done
--;
732 if (cohc
->nbr_active_done
) {
733 dev_dbg(COHC_2_DEV(cohc
), "scheduling tasklet again, new IRQs "
734 "came in while we were scheduling this tasklet\n");
735 if (cohc_chan_conf(cohc
)->priority_high
)
736 tasklet_hi_schedule(&cohc
->tasklet
);
738 tasklet_schedule(&cohc
->tasklet
);
741 spin_unlock_irqrestore(&cohc
->lock
, flags
);
746 spin_unlock_irqrestore(&cohc
->lock
, flags
);
747 dev_err(COHC_2_DEV(cohc
), "[%s] No active dma desc\n", __func__
);
751 /* called from interrupt context */
752 static void dma_tc_handle(struct coh901318_chan
*cohc
)
755 * If the channel is not allocated, then we shouldn't have
756 * any TC interrupts on it.
758 if (!cohc
->allocated
) {
759 dev_err(COHC_2_DEV(cohc
), "spurious interrupt from "
760 "unallocated channel\n");
764 spin_lock(&cohc
->lock
);
767 * When we reach this point, at least one queue item
768 * should have been moved over from cohc->queue to
769 * cohc->active and run to completion, that is why we're
770 * getting a terminal count interrupt is it not?
771 * If you get this BUG() the most probable cause is that
772 * the individual nodes in the lli chain have IRQ enabled,
773 * so check your platform config for lli chain ctrl.
775 BUG_ON(list_empty(&cohc
->active
));
777 cohc
->nbr_active_done
++;
780 * This attempt to take a job from cohc->queue, put it
781 * into cohc->active and start it.
783 if (coh901318_queue_start(cohc
) == NULL
)
786 spin_unlock(&cohc
->lock
);
789 * This tasklet will remove items from cohc->active
790 * and thus terminates them.
792 if (cohc_chan_conf(cohc
)->priority_high
)
793 tasklet_hi_schedule(&cohc
->tasklet
);
795 tasklet_schedule(&cohc
->tasklet
);
799 static irqreturn_t
dma_irq_handler(int irq
, void *dev_id
)
805 struct coh901318_base
*base
= dev_id
;
806 struct coh901318_chan
*cohc
;
807 void __iomem
*virtbase
= base
->virtbase
;
809 status1
= readl(virtbase
+ COH901318_INT_STATUS1
);
810 status2
= readl(virtbase
+ COH901318_INT_STATUS2
);
812 if (unlikely(status1
== 0 && status2
== 0)) {
813 dev_warn(base
->dev
, "spurious DMA IRQ from no channel!\n");
817 /* TODO: consider handle IRQ in tasklet here to
818 * minimize interrupt latency */
820 /* Check the first 32 DMA channels for IRQ */
822 /* Find first bit set, return as a number. */
823 i
= ffs(status1
) - 1;
826 cohc
= &base
->chans
[ch
];
827 spin_lock(&cohc
->lock
);
829 /* Mask off this bit */
830 status1
&= ~(1 << i
);
831 /* Check the individual channel bits */
832 if (test_bit(i
, virtbase
+ COH901318_BE_INT_STATUS1
)) {
833 dev_crit(COHC_2_DEV(cohc
),
834 "DMA bus error on channel %d!\n", ch
);
836 /* Clear BE interrupt */
837 __set_bit(i
, virtbase
+ COH901318_BE_INT_CLEAR1
);
839 /* Caused by TC, really? */
840 if (unlikely(!test_bit(i
, virtbase
+
841 COH901318_TC_INT_STATUS1
))) {
842 dev_warn(COHC_2_DEV(cohc
),
843 "ignoring interrupt not caused by terminal count on channel %d\n", ch
);
844 /* Clear TC interrupt */
846 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR1
);
848 /* Enable powersave if transfer has finished */
849 if (!(readl(virtbase
+ COH901318_CX_STAT
+
850 COH901318_CX_STAT_SPACING
*ch
) &
851 COH901318_CX_STAT_ENABLED
)) {
852 enable_powersave(cohc
);
855 /* Must clear TC interrupt before calling
857 * in case tc_handle initiate a new dma job
859 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR1
);
864 spin_unlock(&cohc
->lock
);
867 /* Check the remaining 32 DMA channels for IRQ */
869 /* Find first bit set, return as a number. */
870 i
= ffs(status2
) - 1;
872 cohc
= &base
->chans
[ch
];
873 spin_lock(&cohc
->lock
);
875 /* Mask off this bit */
876 status2
&= ~(1 << i
);
877 /* Check the individual channel bits */
878 if (test_bit(i
, virtbase
+ COH901318_BE_INT_STATUS2
)) {
879 dev_crit(COHC_2_DEV(cohc
),
880 "DMA bus error on channel %d!\n", ch
);
881 /* Clear BE interrupt */
883 __set_bit(i
, virtbase
+ COH901318_BE_INT_CLEAR2
);
885 /* Caused by TC, really? */
886 if (unlikely(!test_bit(i
, virtbase
+
887 COH901318_TC_INT_STATUS2
))) {
888 dev_warn(COHC_2_DEV(cohc
),
889 "ignoring interrupt not caused by terminal count on channel %d\n", ch
);
890 /* Clear TC interrupt */
891 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR2
);
894 /* Enable powersave if transfer has finished */
895 if (!(readl(virtbase
+ COH901318_CX_STAT
+
896 COH901318_CX_STAT_SPACING
*ch
) &
897 COH901318_CX_STAT_ENABLED
)) {
898 enable_powersave(cohc
);
900 /* Must clear TC interrupt before calling
902 * in case tc_handle initiate a new dma job
904 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR2
);
909 spin_unlock(&cohc
->lock
);
915 static int coh901318_alloc_chan_resources(struct dma_chan
*chan
)
917 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
920 dev_vdbg(COHC_2_DEV(cohc
), "[%s] DMA channel %d\n",
923 if (chan
->client_count
> 1)
926 spin_lock_irqsave(&cohc
->lock
, flags
);
928 coh901318_config(cohc
, NULL
);
931 chan
->completed_cookie
= chan
->cookie
= 1;
933 spin_unlock_irqrestore(&cohc
->lock
, flags
);
939 coh901318_free_chan_resources(struct dma_chan
*chan
)
941 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
942 int channel
= cohc
->id
;
945 spin_lock_irqsave(&cohc
->lock
, flags
);
948 writel(0x00000000U
, cohc
->base
->virtbase
+ COH901318_CX_CFG
+
949 COH901318_CX_CFG_SPACING
*channel
);
950 writel(0x00000000U
, cohc
->base
->virtbase
+ COH901318_CX_CTRL
+
951 COH901318_CX_CTRL_SPACING
*channel
);
955 spin_unlock_irqrestore(&cohc
->lock
, flags
);
957 chan
->device
->device_control(chan
, DMA_TERMINATE_ALL
, 0);
962 coh901318_tx_submit(struct dma_async_tx_descriptor
*tx
)
964 struct coh901318_desc
*cohd
= container_of(tx
, struct coh901318_desc
,
966 struct coh901318_chan
*cohc
= to_coh901318_chan(tx
->chan
);
969 spin_lock_irqsave(&cohc
->lock
, flags
);
971 tx
->cookie
= coh901318_assign_cookie(cohc
, cohd
);
973 coh901318_desc_queue(cohc
, cohd
);
975 spin_unlock_irqrestore(&cohc
->lock
, flags
);
980 static struct dma_async_tx_descriptor
*
981 coh901318_prep_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
982 size_t size
, unsigned long flags
)
984 struct coh901318_lli
*lli
;
985 struct coh901318_desc
*cohd
;
987 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
989 u32 ctrl_last
= cohc_chan_param(cohc
)->ctrl_lli_last
;
992 spin_lock_irqsave(&cohc
->lock
, flg
);
994 dev_vdbg(COHC_2_DEV(cohc
),
995 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
996 __func__
, cohc
->id
, src
, dest
, size
);
998 if (flags
& DMA_PREP_INTERRUPT
)
999 /* Trigger interrupt after last lli */
1000 ctrl_last
|= COH901318_CX_CTRL_TC_IRQ_ENABLE
;
1002 lli_len
= size
>> MAX_DMA_PACKET_SIZE_SHIFT
;
1003 if ((lli_len
<< MAX_DMA_PACKET_SIZE_SHIFT
) < size
)
1006 lli
= coh901318_lli_alloc(&cohc
->base
->pool
, lli_len
);
1011 ret
= coh901318_lli_fill_memcpy(
1012 &cohc
->base
->pool
, lli
, src
, size
, dest
,
1013 cohc_chan_param(cohc
)->ctrl_lli_chained
,
1018 COH_DBG(coh901318_list_print(cohc
, lli
));
1020 /* Pick a descriptor to handle this transfer */
1021 cohd
= coh901318_desc_get(cohc
);
1023 cohd
->flags
= flags
;
1024 cohd
->desc
.tx_submit
= coh901318_tx_submit
;
1026 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1030 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1034 static struct dma_async_tx_descriptor
*
1035 coh901318_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
1036 unsigned int sg_len
, enum dma_transfer_direction direction
,
1037 unsigned long flags
)
1039 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1040 struct coh901318_lli
*lli
;
1041 struct coh901318_desc
*cohd
;
1042 const struct coh901318_params
*params
;
1043 struct scatterlist
*sg
;
1047 u32 ctrl_chained
= cohc_chan_param(cohc
)->ctrl_lli_chained
;
1048 u32 ctrl
= cohc_chan_param(cohc
)->ctrl_lli
;
1049 u32 ctrl_last
= cohc_chan_param(cohc
)->ctrl_lli_last
;
1056 if (sgl
->length
== 0)
1059 spin_lock_irqsave(&cohc
->lock
, flg
);
1061 dev_vdbg(COHC_2_DEV(cohc
), "[%s] sg_len %d dir %d\n",
1062 __func__
, sg_len
, direction
);
1064 if (flags
& DMA_PREP_INTERRUPT
)
1065 /* Trigger interrupt after last lli */
1066 ctrl_last
|= COH901318_CX_CTRL_TC_IRQ_ENABLE
;
1068 params
= cohc_chan_param(cohc
);
1069 config
= params
->config
;
1071 * Add runtime-specific control on top, make
1072 * sure the bits you set per peripheral channel are
1073 * cleared in the default config from the platform.
1075 ctrl_chained
|= cohc
->runtime_ctrl
;
1076 ctrl_last
|= cohc
->runtime_ctrl
;
1077 ctrl
|= cohc
->runtime_ctrl
;
1079 if (direction
== DMA_MEM_TO_DEV
) {
1080 u32 tx_flags
= COH901318_CX_CTRL_PRDD_SOURCE
|
1081 COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE
;
1083 config
|= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY
;
1084 ctrl_chained
|= tx_flags
;
1085 ctrl_last
|= tx_flags
;
1087 } else if (direction
== DMA_DEV_TO_MEM
) {
1088 u32 rx_flags
= COH901318_CX_CTRL_PRDD_DEST
|
1089 COH901318_CX_CTRL_DST_ADDR_INC_ENABLE
;
1091 config
|= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY
;
1092 ctrl_chained
|= rx_flags
;
1093 ctrl_last
|= rx_flags
;
1098 /* The dma only supports transmitting packages up to
1099 * MAX_DMA_PACKET_SIZE. Calculate to total number of
1100 * dma elemts required to send the entire sg list
1102 for_each_sg(sgl
, sg
, sg_len
, i
) {
1103 unsigned int factor
;
1104 size
= sg_dma_len(sg
);
1106 if (size
<= MAX_DMA_PACKET_SIZE
) {
1111 factor
= size
>> MAX_DMA_PACKET_SIZE_SHIFT
;
1112 if ((factor
<< MAX_DMA_PACKET_SIZE_SHIFT
) < size
)
1118 pr_debug("Allocate %d lli:s for this transfer\n", len
);
1119 lli
= coh901318_lli_alloc(&cohc
->base
->pool
, len
);
1124 /* initiate allocated lli list */
1125 ret
= coh901318_lli_fill_sg(&cohc
->base
->pool
, lli
, sgl
, sg_len
,
1126 cohc_dev_addr(cohc
),
1130 direction
, COH901318_CX_CTRL_TC_IRQ_ENABLE
);
1135 COH_DBG(coh901318_list_print(cohc
, lli
));
1137 /* Pick a descriptor to handle this transfer */
1138 cohd
= coh901318_desc_get(cohc
);
1139 cohd
->head_config
= config
;
1141 * Set the default head ctrl for the channel to the one from the
1142 * lli, things may have changed due to odd buffer alignment
1145 cohd
->head_ctrl
= lli
->control
;
1146 cohd
->dir
= direction
;
1147 cohd
->flags
= flags
;
1148 cohd
->desc
.tx_submit
= coh901318_tx_submit
;
1151 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1157 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1162 static enum dma_status
1163 coh901318_tx_status(struct dma_chan
*chan
, dma_cookie_t cookie
,
1164 struct dma_tx_state
*txstate
)
1166 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1167 dma_cookie_t last_used
;
1168 dma_cookie_t last_complete
;
1171 last_complete
= chan
->completed_cookie
;
1172 last_used
= chan
->cookie
;
1174 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
1176 dma_set_tx_state(txstate
, last_complete
, last_used
,
1177 coh901318_get_bytes_left(chan
));
1178 if (ret
== DMA_IN_PROGRESS
&& cohc
->stopped
)
1185 coh901318_issue_pending(struct dma_chan
*chan
)
1187 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1188 unsigned long flags
;
1190 spin_lock_irqsave(&cohc
->lock
, flags
);
1193 * Busy means that pending jobs are already being processed,
1194 * and then there is no point in starting the queue: the
1195 * terminal count interrupt on the channel will take the next
1196 * job on the queue and execute it anyway.
1199 coh901318_queue_start(cohc
);
1201 spin_unlock_irqrestore(&cohc
->lock
, flags
);
1205 * Here we wrap in the runtime dma control interface
1207 struct burst_table
{
1214 static const struct burst_table burst_sizes
[] = {
1219 .reg
= COH901318_CX_CTRL_BURST_COUNT_64_BYTES
,
1225 .reg
= COH901318_CX_CTRL_BURST_COUNT_48_BYTES
,
1231 .reg
= COH901318_CX_CTRL_BURST_COUNT_32_BYTES
,
1237 .reg
= COH901318_CX_CTRL_BURST_COUNT_16_BYTES
,
1243 .reg
= COH901318_CX_CTRL_BURST_COUNT_8_BYTES
,
1249 .reg
= COH901318_CX_CTRL_BURST_COUNT_4_BYTES
,
1255 .reg
= COH901318_CX_CTRL_BURST_COUNT_2_BYTES
,
1261 .reg
= COH901318_CX_CTRL_BURST_COUNT_1_BYTE
,
1265 static void coh901318_dma_set_runtimeconfig(struct dma_chan
*chan
,
1266 struct dma_slave_config
*config
)
1268 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1270 enum dma_slave_buswidth addr_width
;
1272 u32 runtime_ctrl
= 0;
1275 /* We only support mem to per or per to mem transfers */
1276 if (config
->direction
== DMA_DEV_TO_MEM
) {
1277 addr
= config
->src_addr
;
1278 addr_width
= config
->src_addr_width
;
1279 maxburst
= config
->src_maxburst
;
1280 } else if (config
->direction
== DMA_MEM_TO_DEV
) {
1281 addr
= config
->dst_addr
;
1282 addr_width
= config
->dst_addr_width
;
1283 maxburst
= config
->dst_maxburst
;
1285 dev_err(COHC_2_DEV(cohc
), "illegal channel mode\n");
1289 dev_dbg(COHC_2_DEV(cohc
), "configure channel for %d byte transfers\n",
1291 switch (addr_width
) {
1292 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
1294 COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS
|
1295 COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS
;
1297 while (i
< ARRAY_SIZE(burst_sizes
)) {
1298 if (burst_sizes
[i
].burst_8bit
<= maxburst
)
1304 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
1306 COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS
|
1307 COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS
;
1309 while (i
< ARRAY_SIZE(burst_sizes
)) {
1310 if (burst_sizes
[i
].burst_16bit
<= maxburst
)
1316 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
1317 /* Direction doesn't matter here, it's 32/32 bits */
1319 COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS
|
1320 COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS
;
1322 while (i
< ARRAY_SIZE(burst_sizes
)) {
1323 if (burst_sizes
[i
].burst_32bit
<= maxburst
)
1330 dev_err(COHC_2_DEV(cohc
),
1331 "bad runtimeconfig: alien address width\n");
1335 runtime_ctrl
|= burst_sizes
[i
].reg
;
1336 dev_dbg(COHC_2_DEV(cohc
),
1337 "selected burst size %d bytes for address width %d bytes, maxburst %d\n",
1338 burst_sizes
[i
].burst_8bit
, addr_width
, maxburst
);
1340 cohc
->runtime_addr
= addr
;
1341 cohc
->runtime_ctrl
= runtime_ctrl
;
1345 coh901318_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
1348 unsigned long flags
;
1349 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1350 struct coh901318_desc
*cohd
;
1351 void __iomem
*virtbase
= cohc
->base
->virtbase
;
1353 if (cmd
== DMA_SLAVE_CONFIG
) {
1354 struct dma_slave_config
*config
=
1355 (struct dma_slave_config
*) arg
;
1357 coh901318_dma_set_runtimeconfig(chan
, config
);
1361 if (cmd
== DMA_PAUSE
) {
1362 coh901318_pause(chan
);
1366 if (cmd
== DMA_RESUME
) {
1367 coh901318_resume(chan
);
1371 if (cmd
!= DMA_TERMINATE_ALL
)
1374 /* The remainder of this function terminates the transfer */
1375 coh901318_pause(chan
);
1376 spin_lock_irqsave(&cohc
->lock
, flags
);
1378 /* Clear any pending BE or TC interrupt */
1379 if (cohc
->id
< 32) {
1380 writel(1 << cohc
->id
, virtbase
+ COH901318_BE_INT_CLEAR1
);
1381 writel(1 << cohc
->id
, virtbase
+ COH901318_TC_INT_CLEAR1
);
1383 writel(1 << (cohc
->id
- 32), virtbase
+
1384 COH901318_BE_INT_CLEAR2
);
1385 writel(1 << (cohc
->id
- 32), virtbase
+
1386 COH901318_TC_INT_CLEAR2
);
1389 enable_powersave(cohc
);
1391 while ((cohd
= coh901318_first_active_get(cohc
))) {
1392 /* release the lli allocation*/
1393 coh901318_lli_free(&cohc
->base
->pool
, &cohd
->lli
);
1395 /* return desc to free-list */
1396 coh901318_desc_remove(cohd
);
1397 coh901318_desc_free(cohc
, cohd
);
1400 while ((cohd
= coh901318_first_queued(cohc
))) {
1401 /* release the lli allocation*/
1402 coh901318_lli_free(&cohc
->base
->pool
, &cohd
->lli
);
1404 /* return desc to free-list */
1405 coh901318_desc_remove(cohd
);
1406 coh901318_desc_free(cohc
, cohd
);
1410 cohc
->nbr_active_done
= 0;
1413 spin_unlock_irqrestore(&cohc
->lock
, flags
);
1418 void coh901318_base_init(struct dma_device
*dma
, const int *pick_chans
,
1419 struct coh901318_base
*base
)
1423 struct coh901318_chan
*cohc
;
1425 INIT_LIST_HEAD(&dma
->channels
);
1427 for (chans_i
= 0; pick_chans
[chans_i
] != -1; chans_i
+= 2) {
1428 for (i
= pick_chans
[chans_i
]; i
<= pick_chans
[chans_i
+1]; i
++) {
1429 cohc
= &base
->chans
[i
];
1432 cohc
->chan
.device
= dma
;
1435 /* TODO: do we really need this lock if only one
1436 * client is connected to each channel?
1439 spin_lock_init(&cohc
->lock
);
1441 cohc
->nbr_active_done
= 0;
1443 INIT_LIST_HEAD(&cohc
->free
);
1444 INIT_LIST_HEAD(&cohc
->active
);
1445 INIT_LIST_HEAD(&cohc
->queue
);
1447 tasklet_init(&cohc
->tasklet
, dma_tasklet
,
1448 (unsigned long) cohc
);
1450 list_add_tail(&cohc
->chan
.device_node
,
1456 static int __init
coh901318_probe(struct platform_device
*pdev
)
1459 struct coh901318_platform
*pdata
;
1460 struct coh901318_base
*base
;
1462 struct resource
*io
;
1464 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1466 goto err_get_resource
;
1468 /* Map DMA controller registers to virtual memory */
1469 if (request_mem_region(io
->start
,
1471 pdev
->dev
.driver
->name
) == NULL
) {
1473 goto err_request_mem
;
1476 pdata
= pdev
->dev
.platform_data
;
1478 goto err_no_platformdata
;
1480 base
= kmalloc(ALIGN(sizeof(struct coh901318_base
), 4) +
1481 pdata
->max_channels
*
1482 sizeof(struct coh901318_chan
),
1485 goto err_alloc_coh_dma_channels
;
1487 base
->chans
= ((void *)base
) + ALIGN(sizeof(struct coh901318_base
), 4);
1489 base
->virtbase
= ioremap(io
->start
, resource_size(io
));
1490 if (!base
->virtbase
) {
1492 goto err_no_ioremap
;
1495 base
->dev
= &pdev
->dev
;
1496 base
->platform
= pdata
;
1497 spin_lock_init(&base
->pm
.lock
);
1498 base
->pm
.started_channels
= 0;
1500 COH901318_DEBUGFS_ASSIGN(debugfs_dma_base
, base
);
1502 platform_set_drvdata(pdev
, base
);
1504 irq
= platform_get_irq(pdev
, 0);
1508 err
= request_irq(irq
, dma_irq_handler
, IRQF_DISABLED
,
1511 dev_crit(&pdev
->dev
,
1512 "Cannot allocate IRQ for DMA controller!\n");
1513 goto err_request_irq
;
1516 err
= coh901318_pool_create(&base
->pool
, &pdev
->dev
,
1517 sizeof(struct coh901318_lli
),
1520 goto err_pool_create
;
1522 /* init channels for device transfers */
1523 coh901318_base_init(&base
->dma_slave
, base
->platform
->chans_slave
,
1526 dma_cap_zero(base
->dma_slave
.cap_mask
);
1527 dma_cap_set(DMA_SLAVE
, base
->dma_slave
.cap_mask
);
1529 base
->dma_slave
.device_alloc_chan_resources
= coh901318_alloc_chan_resources
;
1530 base
->dma_slave
.device_free_chan_resources
= coh901318_free_chan_resources
;
1531 base
->dma_slave
.device_prep_slave_sg
= coh901318_prep_slave_sg
;
1532 base
->dma_slave
.device_tx_status
= coh901318_tx_status
;
1533 base
->dma_slave
.device_issue_pending
= coh901318_issue_pending
;
1534 base
->dma_slave
.device_control
= coh901318_control
;
1535 base
->dma_slave
.dev
= &pdev
->dev
;
1537 err
= dma_async_device_register(&base
->dma_slave
);
1540 goto err_register_slave
;
1542 /* init channels for memcpy */
1543 coh901318_base_init(&base
->dma_memcpy
, base
->platform
->chans_memcpy
,
1546 dma_cap_zero(base
->dma_memcpy
.cap_mask
);
1547 dma_cap_set(DMA_MEMCPY
, base
->dma_memcpy
.cap_mask
);
1549 base
->dma_memcpy
.device_alloc_chan_resources
= coh901318_alloc_chan_resources
;
1550 base
->dma_memcpy
.device_free_chan_resources
= coh901318_free_chan_resources
;
1551 base
->dma_memcpy
.device_prep_dma_memcpy
= coh901318_prep_memcpy
;
1552 base
->dma_memcpy
.device_tx_status
= coh901318_tx_status
;
1553 base
->dma_memcpy
.device_issue_pending
= coh901318_issue_pending
;
1554 base
->dma_memcpy
.device_control
= coh901318_control
;
1555 base
->dma_memcpy
.dev
= &pdev
->dev
;
1557 * This controller can only access address at even 32bit boundaries,
1560 base
->dma_memcpy
.copy_align
= 2;
1561 err
= dma_async_device_register(&base
->dma_memcpy
);
1564 goto err_register_memcpy
;
1566 dev_info(&pdev
->dev
, "Initialized COH901318 DMA on virtual base 0x%08x\n",
1567 (u32
) base
->virtbase
);
1571 err_register_memcpy
:
1572 dma_async_device_unregister(&base
->dma_slave
);
1574 coh901318_pool_destroy(&base
->pool
);
1576 free_irq(platform_get_irq(pdev
, 0), base
);
1579 iounmap(base
->virtbase
);
1582 err_alloc_coh_dma_channels
:
1583 err_no_platformdata
:
1584 release_mem_region(pdev
->resource
->start
,
1585 resource_size(pdev
->resource
));
1591 static int __exit
coh901318_remove(struct platform_device
*pdev
)
1593 struct coh901318_base
*base
= platform_get_drvdata(pdev
);
1595 dma_async_device_unregister(&base
->dma_memcpy
);
1596 dma_async_device_unregister(&base
->dma_slave
);
1597 coh901318_pool_destroy(&base
->pool
);
1598 free_irq(platform_get_irq(pdev
, 0), base
);
1599 iounmap(base
->virtbase
);
1601 release_mem_region(pdev
->resource
->start
,
1602 resource_size(pdev
->resource
));
1607 static struct platform_driver coh901318_driver
= {
1608 .remove
= __exit_p(coh901318_remove
),
1610 .name
= "coh901318",
1614 int __init
coh901318_init(void)
1616 return platform_driver_probe(&coh901318_driver
, coh901318_probe
);
1618 subsys_initcall(coh901318_init
);
1620 void __exit
coh901318_exit(void)
1622 platform_driver_unregister(&coh901318_driver
);
1624 module_exit(coh901318_exit
);
1626 MODULE_LICENSE("GPL");
1627 MODULE_AUTHOR("Per Friden");