2 * Core driver for the High Speed UART DMA
4 * Copyright (C) 2015 Intel Corporation
5 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
7 * Partially based on the bits found in drivers/tty/serial/mfd.c.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
15 * DMA channel allocation:
16 * 1. Even number chans are used for DMA Read (UART TX), odd chans for DMA
18 * 2. 0/1 channel are assigned to port 0, 2/3 chan to port 1, 4/5 chan to
22 #include <linux/delay.h>
23 #include <linux/dmaengine.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/init.h>
26 #include <linux/module.h>
27 #include <linux/slab.h>
31 #define HSU_DMA_BUSWIDTHS \
32 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
33 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
34 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
35 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
36 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
37 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) | \
38 BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)
40 static inline void hsu_chan_disable(struct hsu_dma_chan
*hsuc
)
42 hsu_chan_writel(hsuc
, HSU_CH_CR
, 0);
45 static inline void hsu_chan_enable(struct hsu_dma_chan
*hsuc
)
47 u32 cr
= HSU_CH_CR_CHA
;
49 if (hsuc
->direction
== DMA_MEM_TO_DEV
)
51 else if (hsuc
->direction
== DMA_DEV_TO_MEM
)
54 hsu_chan_writel(hsuc
, HSU_CH_CR
, cr
);
57 static void hsu_dma_chan_start(struct hsu_dma_chan
*hsuc
)
59 struct dma_slave_config
*config
= &hsuc
->config
;
60 struct hsu_dma_desc
*desc
= hsuc
->desc
;
61 u32 bsr
= 0, mtsr
= 0; /* to shut the compiler up */
62 u32 dcr
= HSU_CH_DCR_CHSOE
| HSU_CH_DCR_CHEI
;
63 unsigned int i
, count
;
65 if (hsuc
->direction
== DMA_MEM_TO_DEV
) {
66 bsr
= config
->dst_maxburst
;
67 mtsr
= config
->src_addr_width
;
68 } else if (hsuc
->direction
== DMA_DEV_TO_MEM
) {
69 bsr
= config
->src_maxburst
;
70 mtsr
= config
->dst_addr_width
;
73 hsu_chan_disable(hsuc
);
75 hsu_chan_writel(hsuc
, HSU_CH_DCR
, 0);
76 hsu_chan_writel(hsuc
, HSU_CH_BSR
, bsr
);
77 hsu_chan_writel(hsuc
, HSU_CH_MTSR
, mtsr
);
80 count
= desc
->nents
- desc
->active
;
81 for (i
= 0; i
< count
&& i
< HSU_DMA_CHAN_NR_DESC
; i
++) {
82 hsu_chan_writel(hsuc
, HSU_CH_DxSAR(i
), desc
->sg
[i
].addr
);
83 hsu_chan_writel(hsuc
, HSU_CH_DxTSR(i
), desc
->sg
[i
].len
);
85 /* Prepare value for DCR */
86 dcr
|= HSU_CH_DCR_DESCA(i
);
87 dcr
|= HSU_CH_DCR_CHTOI(i
); /* timeout bit, see HSU Errata 1 */
91 /* Only for the last descriptor in the chain */
92 dcr
|= HSU_CH_DCR_CHSOD(count
- 1);
93 dcr
|= HSU_CH_DCR_CHDI(count
- 1);
95 hsu_chan_writel(hsuc
, HSU_CH_DCR
, dcr
);
97 hsu_chan_enable(hsuc
);
100 static void hsu_dma_stop_channel(struct hsu_dma_chan
*hsuc
)
102 hsu_chan_disable(hsuc
);
103 hsu_chan_writel(hsuc
, HSU_CH_DCR
, 0);
106 static void hsu_dma_start_channel(struct hsu_dma_chan
*hsuc
)
108 hsu_dma_chan_start(hsuc
);
111 static void hsu_dma_start_transfer(struct hsu_dma_chan
*hsuc
)
113 struct virt_dma_desc
*vdesc
;
115 /* Get the next descriptor */
116 vdesc
= vchan_next_desc(&hsuc
->vchan
);
122 list_del(&vdesc
->node
);
123 hsuc
->desc
= to_hsu_dma_desc(vdesc
);
125 /* Start the channel with a new descriptor */
126 hsu_dma_start_channel(hsuc
);
130 * hsu_dma_get_status() - get DMA channel status
131 * @chip: HSUART DMA chip
132 * @nr: DMA channel number
133 * @status: pointer for DMA Channel Status Register value
136 * The function reads and clears the DMA Channel Status Register, checks
137 * if it was a timeout interrupt and returns a corresponding value.
139 * Caller should provide a valid pointer for the DMA Channel Status
140 * Register value that will be returned in @status.
143 * 1 for DMA timeout status, 0 for other DMA status, or error code for
144 * invalid parameters or no interrupt pending.
146 int hsu_dma_get_status(struct hsu_dma_chip
*chip
, unsigned short nr
,
149 struct hsu_dma_chan
*hsuc
;
154 if (nr
>= chip
->hsu
->nr_channels
)
157 hsuc
= &chip
->hsu
->chan
[nr
];
160 * No matter what situation, need read clear the IRQ status
161 * There is a bug, see Errata 5, HSD 2900918
163 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
164 sr
= hsu_chan_readl(hsuc
, HSU_CH_SR
);
165 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
167 /* Check if any interrupt is pending */
168 sr
&= ~(HSU_CH_SR_DESCE_ANY
| HSU_CH_SR_CDESC_ANY
);
172 /* Timeout IRQ, need wait some time, see Errata 2 */
173 if (sr
& HSU_CH_SR_DESCTO_ANY
)
177 * At this point, at least one of Descriptor Time Out, Channel Error
178 * or Descriptor Done bits must be set. Clear the Descriptor Time Out
179 * bits and if sr is still non-zero, it must be channel error or
180 * descriptor done which are higher priority than timeout and handled
181 * in hsu_dma_do_irq(). Else, it must be a timeout.
183 sr
&= ~HSU_CH_SR_DESCTO_ANY
;
189 EXPORT_SYMBOL_GPL(hsu_dma_get_status
);
192 * hsu_dma_do_irq() - DMA interrupt handler
193 * @chip: HSUART DMA chip
194 * @nr: DMA channel number
195 * @status: Channel Status Register value
198 * This function handles Channel Error and Descriptor Done interrupts.
199 * This function should be called after determining that the DMA interrupt
200 * is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0.
203 * IRQ_NONE for invalid channel number, IRQ_HANDLED otherwise.
205 irqreturn_t
hsu_dma_do_irq(struct hsu_dma_chip
*chip
, unsigned short nr
,
208 struct hsu_dma_chan
*hsuc
;
209 struct hsu_dma_desc
*desc
;
213 if (nr
>= chip
->hsu
->nr_channels
)
216 hsuc
= &chip
->hsu
->chan
[nr
];
218 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
221 if (status
& HSU_CH_SR_CHE
) {
222 desc
->status
= DMA_ERROR
;
223 } else if (desc
->active
< desc
->nents
) {
224 hsu_dma_start_channel(hsuc
);
226 vchan_cookie_complete(&desc
->vdesc
);
227 desc
->status
= DMA_COMPLETE
;
228 hsu_dma_start_transfer(hsuc
);
231 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
235 EXPORT_SYMBOL_GPL(hsu_dma_do_irq
);
237 static struct hsu_dma_desc
*hsu_dma_alloc_desc(unsigned int nents
)
239 struct hsu_dma_desc
*desc
;
241 desc
= kzalloc(sizeof(*desc
), GFP_NOWAIT
);
245 desc
->sg
= kcalloc(nents
, sizeof(*desc
->sg
), GFP_NOWAIT
);
254 static void hsu_dma_desc_free(struct virt_dma_desc
*vdesc
)
256 struct hsu_dma_desc
*desc
= to_hsu_dma_desc(vdesc
);
262 static struct dma_async_tx_descriptor
*hsu_dma_prep_slave_sg(
263 struct dma_chan
*chan
, struct scatterlist
*sgl
,
264 unsigned int sg_len
, enum dma_transfer_direction direction
,
265 unsigned long flags
, void *context
)
267 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
268 struct hsu_dma_desc
*desc
;
269 struct scatterlist
*sg
;
272 desc
= hsu_dma_alloc_desc(sg_len
);
276 for_each_sg(sgl
, sg
, sg_len
, i
) {
277 desc
->sg
[i
].addr
= sg_dma_address(sg
);
278 desc
->sg
[i
].len
= sg_dma_len(sg
);
280 desc
->length
+= sg_dma_len(sg
);
283 desc
->nents
= sg_len
;
284 desc
->direction
= direction
;
285 /* desc->active = 0 by kzalloc */
286 desc
->status
= DMA_IN_PROGRESS
;
288 return vchan_tx_prep(&hsuc
->vchan
, &desc
->vdesc
, flags
);
291 static void hsu_dma_issue_pending(struct dma_chan
*chan
)
293 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
296 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
297 if (vchan_issue_pending(&hsuc
->vchan
) && !hsuc
->desc
)
298 hsu_dma_start_transfer(hsuc
);
299 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
302 static size_t hsu_dma_active_desc_size(struct hsu_dma_chan
*hsuc
)
304 struct hsu_dma_desc
*desc
= hsuc
->desc
;
308 for (i
= desc
->active
; i
< desc
->nents
; i
++)
309 bytes
+= desc
->sg
[i
].len
;
311 i
= HSU_DMA_CHAN_NR_DESC
- 1;
313 bytes
+= hsu_chan_readl(hsuc
, HSU_CH_DxTSR(i
));
319 static enum dma_status
hsu_dma_tx_status(struct dma_chan
*chan
,
320 dma_cookie_t cookie
, struct dma_tx_state
*state
)
322 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
323 struct virt_dma_desc
*vdesc
;
324 enum dma_status status
;
328 status
= dma_cookie_status(chan
, cookie
, state
);
329 if (status
== DMA_COMPLETE
)
332 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
333 vdesc
= vchan_find_desc(&hsuc
->vchan
, cookie
);
334 if (hsuc
->desc
&& cookie
== hsuc
->desc
->vdesc
.tx
.cookie
) {
335 bytes
= hsu_dma_active_desc_size(hsuc
);
336 dma_set_residue(state
, bytes
);
337 status
= hsuc
->desc
->status
;
339 bytes
= to_hsu_dma_desc(vdesc
)->length
;
340 dma_set_residue(state
, bytes
);
342 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
347 static int hsu_dma_slave_config(struct dma_chan
*chan
,
348 struct dma_slave_config
*config
)
350 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
352 /* Check if chan will be configured for slave transfers */
353 if (!is_slave_direction(config
->direction
))
356 memcpy(&hsuc
->config
, config
, sizeof(hsuc
->config
));
361 static int hsu_dma_pause(struct dma_chan
*chan
)
363 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
366 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
367 if (hsuc
->desc
&& hsuc
->desc
->status
== DMA_IN_PROGRESS
) {
368 hsu_chan_disable(hsuc
);
369 hsuc
->desc
->status
= DMA_PAUSED
;
371 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
376 static int hsu_dma_resume(struct dma_chan
*chan
)
378 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
381 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
382 if (hsuc
->desc
&& hsuc
->desc
->status
== DMA_PAUSED
) {
383 hsuc
->desc
->status
= DMA_IN_PROGRESS
;
384 hsu_chan_enable(hsuc
);
386 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
391 static int hsu_dma_terminate_all(struct dma_chan
*chan
)
393 struct hsu_dma_chan
*hsuc
= to_hsu_dma_chan(chan
);
397 spin_lock_irqsave(&hsuc
->vchan
.lock
, flags
);
399 hsu_dma_stop_channel(hsuc
);
401 hsu_dma_desc_free(&hsuc
->desc
->vdesc
);
405 vchan_get_all_descriptors(&hsuc
->vchan
, &head
);
406 spin_unlock_irqrestore(&hsuc
->vchan
.lock
, flags
);
407 vchan_dma_desc_free_list(&hsuc
->vchan
, &head
);
412 static void hsu_dma_free_chan_resources(struct dma_chan
*chan
)
414 vchan_free_chan_resources(to_virt_chan(chan
));
417 int hsu_dma_probe(struct hsu_dma_chip
*chip
)
420 void __iomem
*addr
= chip
->regs
+ chip
->offset
;
424 hsu
= devm_kzalloc(chip
->dev
, sizeof(*hsu
), GFP_KERNEL
);
430 /* Calculate nr_channels from the IO space length */
431 hsu
->nr_channels
= (chip
->length
- chip
->offset
) / HSU_DMA_CHAN_LENGTH
;
433 hsu
->chan
= devm_kcalloc(chip
->dev
, hsu
->nr_channels
,
434 sizeof(*hsu
->chan
), GFP_KERNEL
);
438 INIT_LIST_HEAD(&hsu
->dma
.channels
);
439 for (i
= 0; i
< hsu
->nr_channels
; i
++) {
440 struct hsu_dma_chan
*hsuc
= &hsu
->chan
[i
];
442 hsuc
->vchan
.desc_free
= hsu_dma_desc_free
;
443 vchan_init(&hsuc
->vchan
, &hsu
->dma
);
445 hsuc
->direction
= (i
& 0x1) ? DMA_DEV_TO_MEM
: DMA_MEM_TO_DEV
;
446 hsuc
->reg
= addr
+ i
* HSU_DMA_CHAN_LENGTH
;
449 dma_cap_set(DMA_SLAVE
, hsu
->dma
.cap_mask
);
450 dma_cap_set(DMA_PRIVATE
, hsu
->dma
.cap_mask
);
452 hsu
->dma
.device_free_chan_resources
= hsu_dma_free_chan_resources
;
454 hsu
->dma
.device_prep_slave_sg
= hsu_dma_prep_slave_sg
;
456 hsu
->dma
.device_issue_pending
= hsu_dma_issue_pending
;
457 hsu
->dma
.device_tx_status
= hsu_dma_tx_status
;
459 hsu
->dma
.device_config
= hsu_dma_slave_config
;
460 hsu
->dma
.device_pause
= hsu_dma_pause
;
461 hsu
->dma
.device_resume
= hsu_dma_resume
;
462 hsu
->dma
.device_terminate_all
= hsu_dma_terminate_all
;
464 hsu
->dma
.src_addr_widths
= HSU_DMA_BUSWIDTHS
;
465 hsu
->dma
.dst_addr_widths
= HSU_DMA_BUSWIDTHS
;
466 hsu
->dma
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
467 hsu
->dma
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
469 hsu
->dma
.dev
= chip
->dev
;
471 dma_set_max_seg_size(hsu
->dma
.dev
, HSU_CH_DxTSR_MASK
);
473 ret
= dma_async_device_register(&hsu
->dma
);
477 dev_info(chip
->dev
, "Found HSU DMA, %d channels\n", hsu
->nr_channels
);
480 EXPORT_SYMBOL_GPL(hsu_dma_probe
);
482 int hsu_dma_remove(struct hsu_dma_chip
*chip
)
484 struct hsu_dma
*hsu
= chip
->hsu
;
487 dma_async_device_unregister(&hsu
->dma
);
489 for (i
= 0; i
< hsu
->nr_channels
; i
++) {
490 struct hsu_dma_chan
*hsuc
= &hsu
->chan
[i
];
492 tasklet_kill(&hsuc
->vchan
.task
);
497 EXPORT_SYMBOL_GPL(hsu_dma_remove
);
499 MODULE_LICENSE("GPL v2");
500 MODULE_DESCRIPTION("High Speed UART DMA core driver");
501 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");