2 * Core driver for the Intel integrated DMA 64-bit
4 * Copyright (C) 2015 Intel Corporation
5 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/bitops.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/init.h>
18 #include <linux/module.h>
19 #include <linux/platform_device.h>
20 #include <linux/slab.h>
24 /* Platform driver name */
25 #define DRV_NAME "idma64"
27 /* For now we support only two channels */
28 #define IDMA64_NR_CHAN 2
30 /* ---------------------------------------------------------------------- */
32 static struct device
*chan2dev(struct dma_chan
*chan
)
34 return &chan
->dev
->device
;
37 /* ---------------------------------------------------------------------- */
39 static void idma64_off(struct idma64
*idma64
)
41 unsigned short count
= 100;
43 dma_writel(idma64
, CFG
, 0);
45 channel_clear_bit(idma64
, MASK(XFER
), idma64
->all_chan_mask
);
46 channel_clear_bit(idma64
, MASK(BLOCK
), idma64
->all_chan_mask
);
47 channel_clear_bit(idma64
, MASK(SRC_TRAN
), idma64
->all_chan_mask
);
48 channel_clear_bit(idma64
, MASK(DST_TRAN
), idma64
->all_chan_mask
);
49 channel_clear_bit(idma64
, MASK(ERROR
), idma64
->all_chan_mask
);
53 } while (dma_readl(idma64
, CFG
) & IDMA64_CFG_DMA_EN
&& --count
);
56 static void idma64_on(struct idma64
*idma64
)
58 dma_writel(idma64
, CFG
, IDMA64_CFG_DMA_EN
);
61 /* ---------------------------------------------------------------------- */
63 static void idma64_chan_init(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
65 u32 cfghi
= IDMA64C_CFGH_SRC_PER(1) | IDMA64C_CFGH_DST_PER(0);
68 /* Enforce FIFO drain when channel is suspended */
69 cfglo
|= IDMA64C_CFGL_CH_DRAIN
;
71 /* Set default burst alignment */
72 cfglo
|= IDMA64C_CFGL_DST_BURST_ALIGN
| IDMA64C_CFGL_SRC_BURST_ALIGN
;
74 channel_writel(idma64c
, CFG_LO
, cfglo
);
75 channel_writel(idma64c
, CFG_HI
, cfghi
);
77 /* Enable interrupts */
78 channel_set_bit(idma64
, MASK(XFER
), idma64c
->mask
);
79 channel_set_bit(idma64
, MASK(ERROR
), idma64c
->mask
);
82 * Enforce the controller to be turned on.
84 * The iDMA is turned off in ->probe() and looses context during system
85 * suspend / resume cycle. That's why we have to enable it each time we
91 static void idma64_chan_stop(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
93 channel_clear_bit(idma64
, CH_EN
, idma64c
->mask
);
96 static void idma64_chan_start(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
98 struct idma64_desc
*desc
= idma64c
->desc
;
99 struct idma64_hw_desc
*hw
= &desc
->hw
[0];
101 channel_writeq(idma64c
, SAR
, 0);
102 channel_writeq(idma64c
, DAR
, 0);
104 channel_writel(idma64c
, CTL_HI
, IDMA64C_CTLH_BLOCK_TS(~0UL));
105 channel_writel(idma64c
, CTL_LO
, IDMA64C_CTLL_LLP_S_EN
| IDMA64C_CTLL_LLP_D_EN
);
107 channel_writeq(idma64c
, LLP
, hw
->llp
);
109 channel_set_bit(idma64
, CH_EN
, idma64c
->mask
);
112 static void idma64_stop_transfer(struct idma64_chan
*idma64c
)
114 struct idma64
*idma64
= to_idma64(idma64c
->vchan
.chan
.device
);
116 idma64_chan_stop(idma64
, idma64c
);
119 static void idma64_start_transfer(struct idma64_chan
*idma64c
)
121 struct idma64
*idma64
= to_idma64(idma64c
->vchan
.chan
.device
);
122 struct virt_dma_desc
*vdesc
;
124 /* Get the next descriptor */
125 vdesc
= vchan_next_desc(&idma64c
->vchan
);
127 idma64c
->desc
= NULL
;
131 list_del(&vdesc
->node
);
132 idma64c
->desc
= to_idma64_desc(vdesc
);
134 /* Configure the channel */
135 idma64_chan_init(idma64
, idma64c
);
137 /* Start the channel with a new descriptor */
138 idma64_chan_start(idma64
, idma64c
);
141 /* ---------------------------------------------------------------------- */
143 static void idma64_chan_irq(struct idma64
*idma64
, unsigned short c
,
144 u32 status_err
, u32 status_xfer
)
146 struct idma64_chan
*idma64c
= &idma64
->chan
[c
];
147 struct idma64_desc
*desc
;
150 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
151 desc
= idma64c
->desc
;
153 if (status_err
& (1 << c
)) {
154 dma_writel(idma64
, CLEAR(ERROR
), idma64c
->mask
);
155 desc
->status
= DMA_ERROR
;
156 } else if (status_xfer
& (1 << c
)) {
157 dma_writel(idma64
, CLEAR(XFER
), idma64c
->mask
);
158 desc
->status
= DMA_COMPLETE
;
159 vchan_cookie_complete(&desc
->vdesc
);
160 idma64_start_transfer(idma64c
);
163 /* idma64_start_transfer() updates idma64c->desc */
164 if (idma64c
->desc
== NULL
|| desc
->status
== DMA_ERROR
)
165 idma64_stop_transfer(idma64c
);
167 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
170 static irqreturn_t
idma64_irq(int irq
, void *dev
)
172 struct idma64
*idma64
= dev
;
173 u32 status
= dma_readl(idma64
, STATUS_INT
);
178 dev_vdbg(idma64
->dma
.dev
, "%s: status=%#x\n", __func__
, status
);
180 /* Check if we have any interrupt from the DMA controller */
184 /* Disable interrupts */
185 channel_clear_bit(idma64
, MASK(XFER
), idma64
->all_chan_mask
);
186 channel_clear_bit(idma64
, MASK(ERROR
), idma64
->all_chan_mask
);
188 status_xfer
= dma_readl(idma64
, RAW(XFER
));
189 status_err
= dma_readl(idma64
, RAW(ERROR
));
191 for (i
= 0; i
< idma64
->dma
.chancnt
; i
++)
192 idma64_chan_irq(idma64
, i
, status_err
, status_xfer
);
194 /* Re-enable interrupts */
195 channel_set_bit(idma64
, MASK(XFER
), idma64
->all_chan_mask
);
196 channel_set_bit(idma64
, MASK(ERROR
), idma64
->all_chan_mask
);
201 /* ---------------------------------------------------------------------- */
203 static struct idma64_desc
*idma64_alloc_desc(unsigned int ndesc
)
205 struct idma64_desc
*desc
;
207 desc
= kzalloc(sizeof(*desc
), GFP_NOWAIT
);
211 desc
->hw
= kcalloc(ndesc
, sizeof(*desc
->hw
), GFP_NOWAIT
);
220 static void idma64_desc_free(struct idma64_chan
*idma64c
,
221 struct idma64_desc
*desc
)
223 struct idma64_hw_desc
*hw
;
226 unsigned int i
= desc
->ndesc
;
230 dma_pool_free(idma64c
->pool
, hw
->lli
, hw
->llp
);
238 static void idma64_vdesc_free(struct virt_dma_desc
*vdesc
)
240 struct idma64_chan
*idma64c
= to_idma64_chan(vdesc
->tx
.chan
);
242 idma64_desc_free(idma64c
, to_idma64_desc(vdesc
));
245 static u64
idma64_hw_desc_fill(struct idma64_hw_desc
*hw
,
246 struct dma_slave_config
*config
,
247 enum dma_transfer_direction direction
, u64 llp
)
249 struct idma64_lli
*lli
= hw
->lli
;
251 u32 ctlhi
= IDMA64C_CTLH_BLOCK_TS(hw
->len
);
252 u32 ctllo
= IDMA64C_CTLL_LLP_S_EN
| IDMA64C_CTLL_LLP_D_EN
;
253 u32 src_width
, dst_width
;
255 if (direction
== DMA_MEM_TO_DEV
) {
257 dar
= config
->dst_addr
;
258 ctllo
|= IDMA64C_CTLL_DST_FIX
| IDMA64C_CTLL_SRC_INC
|
260 src_width
= min_t(u32
, 2, __fls(sar
| hw
->len
));
261 dst_width
= __fls(config
->dst_addr_width
);
262 } else { /* DMA_DEV_TO_MEM */
263 sar
= config
->src_addr
;
265 ctllo
|= IDMA64C_CTLL_DST_INC
| IDMA64C_CTLL_SRC_FIX
|
267 src_width
= __fls(config
->src_addr_width
);
268 dst_width
= min_t(u32
, 2, __fls(dar
| hw
->len
));
276 IDMA64C_CTLL_SRC_MSIZE(config
->src_maxburst
) |
277 IDMA64C_CTLL_DST_MSIZE(config
->dst_maxburst
) |
278 IDMA64C_CTLL_DST_WIDTH(dst_width
) |
279 IDMA64C_CTLL_SRC_WIDTH(src_width
);
285 static void idma64_desc_fill(struct idma64_chan
*idma64c
,
286 struct idma64_desc
*desc
)
288 struct dma_slave_config
*config
= &idma64c
->config
;
289 struct idma64_hw_desc
*hw
= &desc
->hw
[desc
->ndesc
- 1];
290 struct idma64_lli
*lli
= hw
->lli
;
292 unsigned int i
= desc
->ndesc
;
294 /* Fill the hardware descriptors and link them to a list */
297 llp
= idma64_hw_desc_fill(hw
, config
, desc
->direction
, llp
);
298 desc
->length
+= hw
->len
;
301 /* Trigger interrupt after last block */
302 lli
->ctllo
|= IDMA64C_CTLL_INT_EN
;
305 static struct dma_async_tx_descriptor
*idma64_prep_slave_sg(
306 struct dma_chan
*chan
, struct scatterlist
*sgl
,
307 unsigned int sg_len
, enum dma_transfer_direction direction
,
308 unsigned long flags
, void *context
)
310 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
311 struct idma64_desc
*desc
;
312 struct scatterlist
*sg
;
315 desc
= idma64_alloc_desc(sg_len
);
319 for_each_sg(sgl
, sg
, sg_len
, i
) {
320 struct idma64_hw_desc
*hw
= &desc
->hw
[i
];
322 /* Allocate DMA capable memory for hardware descriptor */
323 hw
->lli
= dma_pool_alloc(idma64c
->pool
, GFP_NOWAIT
, &hw
->llp
);
326 idma64_desc_free(idma64c
, desc
);
330 hw
->phys
= sg_dma_address(sg
);
331 hw
->len
= sg_dma_len(sg
);
334 desc
->ndesc
= sg_len
;
335 desc
->direction
= direction
;
336 desc
->status
= DMA_IN_PROGRESS
;
338 idma64_desc_fill(idma64c
, desc
);
339 return vchan_tx_prep(&idma64c
->vchan
, &desc
->vdesc
, flags
);
342 static void idma64_issue_pending(struct dma_chan
*chan
)
344 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
347 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
348 if (vchan_issue_pending(&idma64c
->vchan
) && !idma64c
->desc
)
349 idma64_start_transfer(idma64c
);
350 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
353 static size_t idma64_active_desc_size(struct idma64_chan
*idma64c
)
355 struct idma64_desc
*desc
= idma64c
->desc
;
356 struct idma64_hw_desc
*hw
;
357 size_t bytes
= desc
->length
;
358 u64 llp
= channel_readq(idma64c
, LLP
);
359 u32 ctlhi
= channel_readl(idma64c
, CTL_HI
);
367 } while (++i
< desc
->ndesc
);
372 /* The current chunk is not fully transfered yet */
373 bytes
+= desc
->hw
[--i
].len
;
375 return bytes
- IDMA64C_CTLH_BLOCK_TS(ctlhi
);
378 static enum dma_status
idma64_tx_status(struct dma_chan
*chan
,
379 dma_cookie_t cookie
, struct dma_tx_state
*state
)
381 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
382 struct virt_dma_desc
*vdesc
;
383 enum dma_status status
;
387 status
= dma_cookie_status(chan
, cookie
, state
);
388 if (status
== DMA_COMPLETE
)
391 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
392 vdesc
= vchan_find_desc(&idma64c
->vchan
, cookie
);
393 if (idma64c
->desc
&& cookie
== idma64c
->desc
->vdesc
.tx
.cookie
) {
394 bytes
= idma64_active_desc_size(idma64c
);
395 dma_set_residue(state
, bytes
);
396 status
= idma64c
->desc
->status
;
398 bytes
= to_idma64_desc(vdesc
)->length
;
399 dma_set_residue(state
, bytes
);
401 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
406 static void convert_burst(u32
*maxburst
)
409 *maxburst
= __fls(*maxburst
);
414 static int idma64_slave_config(struct dma_chan
*chan
,
415 struct dma_slave_config
*config
)
417 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
419 /* Check if chan will be configured for slave transfers */
420 if (!is_slave_direction(config
->direction
))
423 memcpy(&idma64c
->config
, config
, sizeof(idma64c
->config
));
425 convert_burst(&idma64c
->config
.src_maxburst
);
426 convert_burst(&idma64c
->config
.dst_maxburst
);
431 static void idma64_chan_deactivate(struct idma64_chan
*idma64c
)
433 unsigned short count
= 100;
436 cfglo
= channel_readl(idma64c
, CFG_LO
);
437 channel_writel(idma64c
, CFG_LO
, cfglo
| IDMA64C_CFGL_CH_SUSP
);
440 cfglo
= channel_readl(idma64c
, CFG_LO
);
441 } while (!(cfglo
& IDMA64C_CFGL_FIFO_EMPTY
) && --count
);
444 static void idma64_chan_activate(struct idma64_chan
*idma64c
)
448 cfglo
= channel_readl(idma64c
, CFG_LO
);
449 channel_writel(idma64c
, CFG_LO
, cfglo
& ~IDMA64C_CFGL_CH_SUSP
);
452 static int idma64_pause(struct dma_chan
*chan
)
454 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
457 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
458 if (idma64c
->desc
&& idma64c
->desc
->status
== DMA_IN_PROGRESS
) {
459 idma64_chan_deactivate(idma64c
);
460 idma64c
->desc
->status
= DMA_PAUSED
;
462 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
467 static int idma64_resume(struct dma_chan
*chan
)
469 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
472 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
473 if (idma64c
->desc
&& idma64c
->desc
->status
== DMA_PAUSED
) {
474 idma64c
->desc
->status
= DMA_IN_PROGRESS
;
475 idma64_chan_activate(idma64c
);
477 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
482 static int idma64_terminate_all(struct dma_chan
*chan
)
484 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
488 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
489 idma64_chan_deactivate(idma64c
);
490 idma64_stop_transfer(idma64c
);
492 idma64_vdesc_free(&idma64c
->desc
->vdesc
);
493 idma64c
->desc
= NULL
;
495 vchan_get_all_descriptors(&idma64c
->vchan
, &head
);
496 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
498 vchan_dma_desc_free_list(&idma64c
->vchan
, &head
);
502 static int idma64_alloc_chan_resources(struct dma_chan
*chan
)
504 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
506 /* Create a pool of consistent memory blocks for hardware descriptors */
507 idma64c
->pool
= dma_pool_create(dev_name(chan2dev(chan
)),
509 sizeof(struct idma64_lli
), 8, 0);
510 if (!idma64c
->pool
) {
511 dev_err(chan2dev(chan
), "No memory for descriptors\n");
518 static void idma64_free_chan_resources(struct dma_chan
*chan
)
520 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
522 vchan_free_chan_resources(to_virt_chan(chan
));
523 dma_pool_destroy(idma64c
->pool
);
524 idma64c
->pool
= NULL
;
527 /* ---------------------------------------------------------------------- */
529 #define IDMA64_BUSWIDTHS \
530 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
531 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
532 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
534 static int idma64_probe(struct idma64_chip
*chip
)
536 struct idma64
*idma64
;
537 unsigned short nr_chan
= IDMA64_NR_CHAN
;
541 idma64
= devm_kzalloc(chip
->dev
, sizeof(*idma64
), GFP_KERNEL
);
545 idma64
->regs
= chip
->regs
;
546 chip
->idma64
= idma64
;
548 idma64
->chan
= devm_kcalloc(chip
->dev
, nr_chan
, sizeof(*idma64
->chan
),
553 idma64
->all_chan_mask
= (1 << nr_chan
) - 1;
555 /* Turn off iDMA controller */
558 ret
= devm_request_irq(chip
->dev
, chip
->irq
, idma64_irq
, IRQF_SHARED
,
559 dev_name(chip
->dev
), idma64
);
563 INIT_LIST_HEAD(&idma64
->dma
.channels
);
564 for (i
= 0; i
< nr_chan
; i
++) {
565 struct idma64_chan
*idma64c
= &idma64
->chan
[i
];
567 idma64c
->vchan
.desc_free
= idma64_vdesc_free
;
568 vchan_init(&idma64c
->vchan
, &idma64
->dma
);
570 idma64c
->regs
= idma64
->regs
+ i
* IDMA64_CH_LENGTH
;
571 idma64c
->mask
= BIT(i
);
574 dma_cap_set(DMA_SLAVE
, idma64
->dma
.cap_mask
);
575 dma_cap_set(DMA_PRIVATE
, idma64
->dma
.cap_mask
);
577 idma64
->dma
.device_alloc_chan_resources
= idma64_alloc_chan_resources
;
578 idma64
->dma
.device_free_chan_resources
= idma64_free_chan_resources
;
580 idma64
->dma
.device_prep_slave_sg
= idma64_prep_slave_sg
;
582 idma64
->dma
.device_issue_pending
= idma64_issue_pending
;
583 idma64
->dma
.device_tx_status
= idma64_tx_status
;
585 idma64
->dma
.device_config
= idma64_slave_config
;
586 idma64
->dma
.device_pause
= idma64_pause
;
587 idma64
->dma
.device_resume
= idma64_resume
;
588 idma64
->dma
.device_terminate_all
= idma64_terminate_all
;
590 idma64
->dma
.src_addr_widths
= IDMA64_BUSWIDTHS
;
591 idma64
->dma
.dst_addr_widths
= IDMA64_BUSWIDTHS
;
592 idma64
->dma
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
593 idma64
->dma
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
595 idma64
->dma
.dev
= chip
->dev
;
597 ret
= dma_async_device_register(&idma64
->dma
);
601 dev_info(chip
->dev
, "Found Intel integrated DMA 64-bit\n");
605 static int idma64_remove(struct idma64_chip
*chip
)
607 struct idma64
*idma64
= chip
->idma64
;
610 dma_async_device_unregister(&idma64
->dma
);
613 * Explicitly call devm_request_irq() to avoid the side effects with
614 * the scheduled tasklets.
616 devm_free_irq(chip
->dev
, chip
->irq
, idma64
);
618 for (i
= 0; i
< idma64
->dma
.chancnt
; i
++) {
619 struct idma64_chan
*idma64c
= &idma64
->chan
[i
];
621 tasklet_kill(&idma64c
->vchan
.task
);
627 /* ---------------------------------------------------------------------- */
629 static int idma64_platform_probe(struct platform_device
*pdev
)
631 struct idma64_chip
*chip
;
632 struct device
*dev
= &pdev
->dev
;
633 struct resource
*mem
;
636 chip
= devm_kzalloc(dev
, sizeof(*chip
), GFP_KERNEL
);
640 chip
->irq
= platform_get_irq(pdev
, 0);
644 mem
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
645 chip
->regs
= devm_ioremap_resource(dev
, mem
);
646 if (IS_ERR(chip
->regs
))
647 return PTR_ERR(chip
->regs
);
649 ret
= dma_coerce_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
655 ret
= idma64_probe(chip
);
659 platform_set_drvdata(pdev
, chip
);
663 static int idma64_platform_remove(struct platform_device
*pdev
)
665 struct idma64_chip
*chip
= platform_get_drvdata(pdev
);
667 return idma64_remove(chip
);
670 #ifdef CONFIG_PM_SLEEP
672 static int idma64_pm_suspend(struct device
*dev
)
674 struct platform_device
*pdev
= to_platform_device(dev
);
675 struct idma64_chip
*chip
= platform_get_drvdata(pdev
);
677 idma64_off(chip
->idma64
);
681 static int idma64_pm_resume(struct device
*dev
)
683 struct platform_device
*pdev
= to_platform_device(dev
);
684 struct idma64_chip
*chip
= platform_get_drvdata(pdev
);
686 idma64_on(chip
->idma64
);
690 #endif /* CONFIG_PM_SLEEP */
692 static const struct dev_pm_ops idma64_dev_pm_ops
= {
693 SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend
, idma64_pm_resume
)
696 static struct platform_driver idma64_platform_driver
= {
697 .probe
= idma64_platform_probe
,
698 .remove
= idma64_platform_remove
,
701 .pm
= &idma64_dev_pm_ops
,
705 module_platform_driver(idma64_platform_driver
);
707 MODULE_LICENSE("GPL v2");
708 MODULE_DESCRIPTION("iDMA64 core driver");
709 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
710 MODULE_ALIAS("platform:" DRV_NAME
);