projects / deliverable / linux.git / blob
? search:


e5378b5741be6a811133ad3a91a81f86a3cce8df
[deliverable/linux.git] / drivers / dma / dw_dmac.c
1 /*
2 * Core driver for the Synopsys DesignWare DMA Controller
3 *
4 * Copyright (C) 2007-2008 Atmel Corporation
5 * Copyright (C) 2010-2011 ST Microelectronics
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12 #include <linux/bitops.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dmaengine.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/of.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25
26 #include "dw_dmac_regs.h"
27 #include "dmaengine.h"
28
29 /*
30 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
31 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
32 * of which use ARM any more). See the "Databook" from Synopsys for
33 * information beyond what licensees probably provide.
34 *
35 * The driver has currently been tested only with the Atmel AT32AP7000,
36 * which does not support descriptor writeback.
37 */
38
39 static inline unsigned int dwc_get_dms(struct dw_dma_slave *slave)
40 {
41 return slave ? slave->dst_master : 0;
42 }
43
44 static inline unsigned int dwc_get_sms(struct dw_dma_slave *slave)
45 {
46 return slave ? slave->src_master : 1;
47 }
48
49 #define DWC_DEFAULT_CTLLO(_chan) ({ \
50 struct dw_dma_slave *__slave = (_chan->private); \
51 struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \
52 struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
53 bool _is_slave = is_slave_direction(_dwc->direction); \
54 int _dms = dwc_get_dms(__slave); \
55 int _sms = dwc_get_sms(__slave); \
56 u8 _smsize = _is_slave ? _sconfig->src_maxburst : \
57 DW_DMA_MSIZE_16; \
58 u8 _dmsize = _is_slave ? _sconfig->dst_maxburst : \
59 DW_DMA_MSIZE_16; \
60 \
61 (DWC_CTLL_DST_MSIZE(_dmsize) \
62 | DWC_CTLL_SRC_MSIZE(_smsize) \
63 | DWC_CTLL_LLP_D_EN \
64 | DWC_CTLL_LLP_S_EN \
65 | DWC_CTLL_DMS(_dms) \
66 | DWC_CTLL_SMS(_sms)); \
67 })
68
69 /*
70 * Number of descriptors to allocate for each channel. This should be
71 * made configurable somehow; preferably, the clients (at least the
72 * ones using slave transfers) should be able to give us a hint.
73 */
74 #define NR_DESCS_PER_CHANNEL 64
75
76 /*----------------------------------------------------------------------*/
77
78 /*
79 * Because we're not relying on writeback from the controller (it may not
80 * even be configured into the core!) we don't need to use dma_pool. These
81 * descriptors -- and associated data -- are cacheable. We do need to make
82 * sure their dcache entries are written back before handing them off to
83 * the controller, though.
84 */
85
86 static struct device *chan2dev(struct dma_chan *chan)
87 {
88 return &chan->dev->device;
89 }
90 static struct device *chan2parent(struct dma_chan *chan)
91 {
92 return chan->dev->device.parent;
93 }
94
95 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
96 {
97 return to_dw_desc(dwc->active_list.next);
98 }
99
100 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
101 {
102 struct dw_desc *desc, *_desc;
103 struct dw_desc *ret = NULL;
104 unsigned int i = 0;
105 unsigned long flags;
106
107 spin_lock_irqsave(&dwc->lock, flags);
108 list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
109 i++;
110 if (async_tx_test_ack(&desc->txd)) {
111 list_del(&desc->desc_node);
112 ret = desc;
113 break;
114 }
115 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
116 }
117 spin_unlock_irqrestore(&dwc->lock, flags);
118
119 dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
120
121 return ret;
122 }
123
124 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
125 {
126 struct dw_desc *child;
127
128 list_for_each_entry(child, &desc->tx_list, desc_node)
129 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
130 child->txd.phys, sizeof(child->lli),
131 DMA_TO_DEVICE);
132 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
133 desc->txd.phys, sizeof(desc->lli),
134 DMA_TO_DEVICE);
135 }
136
137 /*
138 * Move a descriptor, including any children, to the free list.
139 * `desc' must not be on any lists.
140 */
141 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
142 {
143 unsigned long flags;
144
145 if (desc) {
146 struct dw_desc *child;
147
148 dwc_sync_desc_for_cpu(dwc, desc);
149
150 spin_lock_irqsave(&dwc->lock, flags);
151 list_for_each_entry(child, &desc->tx_list, desc_node)
152 dev_vdbg(chan2dev(&dwc->chan),
153 "moving child desc %p to freelist\n",
154 child);
155 list_splice_init(&desc->tx_list, &dwc->free_list);
156 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
157 list_add(&desc->desc_node, &dwc->free_list);
158 spin_unlock_irqrestore(&dwc->lock, flags);
159 }
160 }
161
162 static void dwc_initialize(struct dw_dma_chan *dwc)
163 {
164 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
165 struct dw_dma_slave *dws = dwc->chan.private;
166 u32 cfghi = DWC_CFGH_FIFO_MODE;
167 u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
168
169 if (dwc->initialized == true)
170 return;
171
172 if (dws) {
173 /*
174 * We need controller-specific data to set up slave
175 * transfers.
176 */
177 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
178
179 cfghi = dws->cfg_hi;
180 cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
181 } else {
182 if (dwc->direction == DMA_MEM_TO_DEV)
183 cfghi = DWC_CFGH_DST_PER(dwc->dma_sconfig.slave_id);
184 else if (dwc->direction == DMA_DEV_TO_MEM)
185 cfghi = DWC_CFGH_SRC_PER(dwc->dma_sconfig.slave_id);
186 }
187
188 channel_writel(dwc, CFG_LO, cfglo);
189 channel_writel(dwc, CFG_HI, cfghi);
190
191 /* Enable interrupts */
192 channel_set_bit(dw, MASK.XFER, dwc->mask);
193 channel_set_bit(dw, MASK.ERROR, dwc->mask);
194
195 dwc->initialized = true;
196 }
197
198 /*----------------------------------------------------------------------*/
199
200 static inline unsigned int dwc_fast_fls(unsigned long long v)
201 {
202 /*
203 * We can be a lot more clever here, but this should take care
204 * of the most common optimization.
205 */
206 if (!(v & 7))
207 return 3;
208 else if (!(v & 3))
209 return 2;
210 else if (!(v & 1))
211 return 1;
212 return 0;
213 }
214
215 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
216 {
217 dev_err(chan2dev(&dwc->chan),
218 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
219 channel_readl(dwc, SAR),
220 channel_readl(dwc, DAR),
221 channel_readl(dwc, LLP),
222 channel_readl(dwc, CTL_HI),
223 channel_readl(dwc, CTL_LO));
224 }
225
226 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
227 {
228 channel_clear_bit(dw, CH_EN, dwc->mask);
229 while (dma_readl(dw, CH_EN) & dwc->mask)
230 cpu_relax();
231 }
232
233 /*----------------------------------------------------------------------*/
234
235 /* Perform single block transfer */
236 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
237 struct dw_desc *desc)
238 {
239 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
240 u32 ctllo;
241
242 /* Software emulation of LLP mode relies on interrupts to continue
243 * multi block transfer. */
244 ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;
245
246 channel_writel(dwc, SAR, desc->lli.sar);
247 channel_writel(dwc, DAR, desc->lli.dar);
248 channel_writel(dwc, CTL_LO, ctllo);
249 channel_writel(dwc, CTL_HI, desc->lli.ctlhi);
250 channel_set_bit(dw, CH_EN, dwc->mask);
251
252 /* Move pointer to next descriptor */
253 dwc->tx_node_active = dwc->tx_node_active->next;
254 }
255
256 /* Called with dwc->lock held and bh disabled */
257 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
258 {
259 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
260 unsigned long was_soft_llp;
261
262 /* ASSERT: channel is idle */
263 if (dma_readl(dw, CH_EN) & dwc->mask) {
264 dev_err(chan2dev(&dwc->chan),
265 "BUG: Attempted to start non-idle channel\n");
266 dwc_dump_chan_regs(dwc);
267
268 /* The tasklet will hopefully advance the queue... */
269 return;
270 }
271
272 if (dwc->nollp) {
273 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
274 &dwc->flags);
275 if (was_soft_llp) {
276 dev_err(chan2dev(&dwc->chan),
277 "BUG: Attempted to start new LLP transfer "
278 "inside ongoing one\n");
279 return;
280 }
281
282 dwc_initialize(dwc);
283
284 dwc->tx_list = &first->tx_list;
285 dwc->tx_node_active = &first->tx_list;
286
287 dwc_do_single_block(dwc, first);
288
289 return;
290 }
291
292 dwc_initialize(dwc);
293
294 channel_writel(dwc, LLP, first->txd.phys);
295 channel_writel(dwc, CTL_LO,
296 DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
297 channel_writel(dwc, CTL_HI, 0);
298 channel_set_bit(dw, CH_EN, dwc->mask);
299 }
300
301 /*----------------------------------------------------------------------*/
302
303 static void
304 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
305 bool callback_required)
306 {
307 dma_async_tx_callback callback = NULL;
308 void *param = NULL;
309 struct dma_async_tx_descriptor *txd = &desc->txd;
310 struct dw_desc *child;
311 unsigned long flags;
312
313 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
314
315 spin_lock_irqsave(&dwc->lock, flags);
316 dma_cookie_complete(txd);
317 if (callback_required) {
318 callback = txd->callback;
319 param = txd->callback_param;
320 }
321
322 dwc_sync_desc_for_cpu(dwc, desc);
323
324 /* async_tx_ack */
325 list_for_each_entry(child, &desc->tx_list, desc_node)
326 async_tx_ack(&child->txd);
327 async_tx_ack(&desc->txd);
328
329 list_splice_init(&desc->tx_list, &dwc->free_list);
330 list_move(&desc->desc_node, &dwc->free_list);
331
332 if (!is_slave_direction(dwc->direction)) {
333 struct device *parent = chan2parent(&dwc->chan);
334 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
335 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
336 dma_unmap_single(parent, desc->lli.dar,
337 desc->len, DMA_FROM_DEVICE);
338 else
339 dma_unmap_page(parent, desc->lli.dar,
340 desc->len, DMA_FROM_DEVICE);
341 }
342 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
343 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
344 dma_unmap_single(parent, desc->lli.sar,
345 desc->len, DMA_TO_DEVICE);
346 else
347 dma_unmap_page(parent, desc->lli.sar,
348 desc->len, DMA_TO_DEVICE);
349 }
350 }
351
352 spin_unlock_irqrestore(&dwc->lock, flags);
353
354 if (callback)
355 callback(param);
356 }
357
358 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
359 {
360 struct dw_desc *desc, *_desc;
361 LIST_HEAD(list);
362 unsigned long flags;
363
364 spin_lock_irqsave(&dwc->lock, flags);
365 if (dma_readl(dw, CH_EN) & dwc->mask) {
366 dev_err(chan2dev(&dwc->chan),
367 "BUG: XFER bit set, but channel not idle!\n");
368
369 /* Try to continue after resetting the channel... */
370 dwc_chan_disable(dw, dwc);
371 }
372
373 /*
374 * Submit queued descriptors ASAP, i.e. before we go through
375 * the completed ones.
376 */
377 list_splice_init(&dwc->active_list, &list);
378 if (!list_empty(&dwc->queue)) {
379 list_move(dwc->queue.next, &dwc->active_list);
380 dwc_dostart(dwc, dwc_first_active(dwc));
381 }
382
383 spin_unlock_irqrestore(&dwc->lock, flags);
384
385 list_for_each_entry_safe(desc, _desc, &list, desc_node)
386 dwc_descriptor_complete(dwc, desc, true);
387 }
388
389 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
390 {
391 dma_addr_t llp;
392 struct dw_desc *desc, *_desc;
393 struct dw_desc *child;
394 u32 status_xfer;
395 unsigned long flags;
396
397 spin_lock_irqsave(&dwc->lock, flags);
398 llp = channel_readl(dwc, LLP);
399 status_xfer = dma_readl(dw, RAW.XFER);
400
401 if (status_xfer & dwc->mask) {
402 /* Everything we've submitted is done */
403 dma_writel(dw, CLEAR.XFER, dwc->mask);
404 spin_unlock_irqrestore(&dwc->lock, flags);
405
406 dwc_complete_all(dw, dwc);
407 return;
408 }
409
410 if (list_empty(&dwc->active_list)) {
411 spin_unlock_irqrestore(&dwc->lock, flags);
412 return;
413 }
414
415 dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__,
416 (unsigned long long)llp);
417
418 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
419 /* check first descriptors addr */
420 if (desc->txd.phys == llp) {
421 spin_unlock_irqrestore(&dwc->lock, flags);
422 return;
423 }
424
425 /* check first descriptors llp */
426 if (desc->lli.llp == llp) {
427 /* This one is currently in progress */
428 spin_unlock_irqrestore(&dwc->lock, flags);
429 return;
430 }
431
432 list_for_each_entry(child, &desc->tx_list, desc_node)
433 if (child->lli.llp == llp) {
434 /* Currently in progress */
435 spin_unlock_irqrestore(&dwc->lock, flags);
436 return;
437 }
438
439 /*
440 * No descriptors so far seem to be in progress, i.e.
441 * this one must be done.
442 */
443 spin_unlock_irqrestore(&dwc->lock, flags);
444 dwc_descriptor_complete(dwc, desc, true);
445 spin_lock_irqsave(&dwc->lock, flags);
446 }
447
448 dev_err(chan2dev(&dwc->chan),
449 "BUG: All descriptors done, but channel not idle!\n");
450
451 /* Try to continue after resetting the channel... */
452 dwc_chan_disable(dw, dwc);
453
454 if (!list_empty(&dwc->queue)) {
455 list_move(dwc->queue.next, &dwc->active_list);
456 dwc_dostart(dwc, dwc_first_active(dwc));
457 }
458 spin_unlock_irqrestore(&dwc->lock, flags);
459 }
460
461 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
462 {
463 dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
464 lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
465 }
466
467 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
468 {
469 struct dw_desc *bad_desc;
470 struct dw_desc *child;
471 unsigned long flags;
472
473 dwc_scan_descriptors(dw, dwc);
474
475 spin_lock_irqsave(&dwc->lock, flags);
476
477 /*
478 * The descriptor currently at the head of the active list is
479 * borked. Since we don't have any way to report errors, we'll
480 * just have to scream loudly and try to carry on.
481 */
482 bad_desc = dwc_first_active(dwc);
483 list_del_init(&bad_desc->desc_node);
484 list_move(dwc->queue.next, dwc->active_list.prev);
485
486 /* Clear the error flag and try to restart the controller */
487 dma_writel(dw, CLEAR.ERROR, dwc->mask);
488 if (!list_empty(&dwc->active_list))
489 dwc_dostart(dwc, dwc_first_active(dwc));
490
491 /*
492 * WARN may seem harsh, but since this only happens
493 * when someone submits a bad physical address in a
494 * descriptor, we should consider ourselves lucky that the
495 * controller flagged an error instead of scribbling over
496 * random memory locations.
497 */
498 dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
499 " cookie: %d\n", bad_desc->txd.cookie);
500 dwc_dump_lli(dwc, &bad_desc->lli);
501 list_for_each_entry(child, &bad_desc->tx_list, desc_node)
502 dwc_dump_lli(dwc, &child->lli);
503
504 spin_unlock_irqrestore(&dwc->lock, flags);
505
506 /* Pretend the descriptor completed successfully */
507 dwc_descriptor_complete(dwc, bad_desc, true);
508 }
509
510 /* --------------------- Cyclic DMA API extensions -------------------- */
511
512 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
513 {
514 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
515 return channel_readl(dwc, SAR);
516 }
517 EXPORT_SYMBOL(dw_dma_get_src_addr);
518
519 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
520 {
521 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
522 return channel_readl(dwc, DAR);
523 }
524 EXPORT_SYMBOL(dw_dma_get_dst_addr);
525
526 /* called with dwc->lock held and all DMAC interrupts disabled */
527 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
528 u32 status_err, u32 status_xfer)
529 {
530 unsigned long flags;
531
532 if (dwc->mask) {
533 void (*callback)(void *param);
534 void *callback_param;
535
536 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
537 channel_readl(dwc, LLP));
538
539 callback = dwc->cdesc->period_callback;
540 callback_param = dwc->cdesc->period_callback_param;
541
542 if (callback)
543 callback(callback_param);
544 }
545
546 /*
547 * Error and transfer complete are highly unlikely, and will most
548 * likely be due to a configuration error by the user.
549 */
550 if (unlikely(status_err & dwc->mask) ||
551 unlikely(status_xfer & dwc->mask)) {
552 int i;
553
554 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
555 "interrupt, stopping DMA transfer\n",
556 status_xfer ? "xfer" : "error");
557
558 spin_lock_irqsave(&dwc->lock, flags);
559
560 dwc_dump_chan_regs(dwc);
561
562 dwc_chan_disable(dw, dwc);
563
564 /* make sure DMA does not restart by loading a new list */
565 channel_writel(dwc, LLP, 0);
566 channel_writel(dwc, CTL_LO, 0);
567 channel_writel(dwc, CTL_HI, 0);
568
569 dma_writel(dw, CLEAR.ERROR, dwc->mask);
570 dma_writel(dw, CLEAR.XFER, dwc->mask);
571
572 for (i = 0; i < dwc->cdesc->periods; i++)
573 dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
574
575 spin_unlock_irqrestore(&dwc->lock, flags);
576 }
577 }
578
579 /* ------------------------------------------------------------------------- */
580
581 static void dw_dma_tasklet(unsigned long data)
582 {
583 struct dw_dma *dw = (struct dw_dma *)data;
584 struct dw_dma_chan *dwc;
585 u32 status_xfer;
586 u32 status_err;
587 int i;
588
589 status_xfer = dma_readl(dw, RAW.XFER);
590 status_err = dma_readl(dw, RAW.ERROR);
591
592 dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
593
594 for (i = 0; i < dw->dma.chancnt; i++) {
595 dwc = &dw->chan[i];
596 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
597 dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
598 else if (status_err & (1 << i))
599 dwc_handle_error(dw, dwc);
600 else if (status_xfer & (1 << i)) {
601 unsigned long flags;
602
603 spin_lock_irqsave(&dwc->lock, flags);
604 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
605 if (dwc->tx_node_active != dwc->tx_list) {
606 struct dw_desc *desc =
607 to_dw_desc(dwc->tx_node_active);
608
609 dma_writel(dw, CLEAR.XFER, dwc->mask);
610
611 dwc_do_single_block(dwc, desc);
612
613 spin_unlock_irqrestore(&dwc->lock, flags);
614 continue;
615 }
616 /* we are done here */
617 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
618 }
619 spin_unlock_irqrestore(&dwc->lock, flags);
620
621 dwc_scan_descriptors(dw, dwc);
622 }
623 }
624
625 /*
626 * Re-enable interrupts.
627 */
628 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
629 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
630 }
631
632 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
633 {
634 struct dw_dma *dw = dev_id;
635 u32 status;
636
637 dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__,
638 dma_readl(dw, STATUS_INT));
639
640 /*
641 * Just disable the interrupts. We'll turn them back on in the
642 * softirq handler.
643 */
644 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
645 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
646
647 status = dma_readl(dw, STATUS_INT);
648 if (status) {
649 dev_err(dw->dma.dev,
650 "BUG: Unexpected interrupts pending: 0x%x\n",
651 status);
652
653 /* Try to recover */
654 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
655 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
656 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
657 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
658 }
659
660 tasklet_schedule(&dw->tasklet);
661
662 return IRQ_HANDLED;
663 }
664
665 /*----------------------------------------------------------------------*/
666
667 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
668 {
669 struct dw_desc *desc = txd_to_dw_desc(tx);
670 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
671 dma_cookie_t cookie;
672 unsigned long flags;
673
674 spin_lock_irqsave(&dwc->lock, flags);
675 cookie = dma_cookie_assign(tx);
676
677 /*
678 * REVISIT: We should attempt to chain as many descriptors as
679 * possible, perhaps even appending to those already submitted
680 * for DMA. But this is hard to do in a race-free manner.
681 */
682 if (list_empty(&dwc->active_list)) {
683 dev_vdbg(chan2dev(tx->chan), "%s: started %u\n", __func__,
684 desc->txd.cookie);
685 list_add_tail(&desc->desc_node, &dwc->active_list);
686 dwc_dostart(dwc, dwc_first_active(dwc));
687 } else {
688 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__,
689 desc->txd.cookie);
690
691 list_add_tail(&desc->desc_node, &dwc->queue);
692 }
693
694 spin_unlock_irqrestore(&dwc->lock, flags);
695
696 return cookie;
697 }
698
699 static struct dma_async_tx_descriptor *
700 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
701 size_t len, unsigned long flags)
702 {
703 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
704 struct dw_dma_slave *dws = chan->private;
705 struct dw_desc *desc;
706 struct dw_desc *first;
707 struct dw_desc *prev;
708 size_t xfer_count;
709 size_t offset;
710 unsigned int src_width;
711 unsigned int dst_width;
712 unsigned int data_width;
713 u32 ctllo;
714
715 dev_vdbg(chan2dev(chan),
716 "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__,
717 (unsigned long long)dest, (unsigned long long)src,
718 len, flags);
719
720 if (unlikely(!len)) {
721 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
722 return NULL;
723 }
724
725 dwc->direction = DMA_MEM_TO_MEM;
726
727 data_width = min_t(unsigned int, dwc->dw->data_width[dwc_get_sms(dws)],
728 dwc->dw->data_width[dwc_get_dms(dws)]);
729
730 src_width = dst_width = min_t(unsigned int, data_width,
731 dwc_fast_fls(src | dest | len));
732
733 ctllo = DWC_DEFAULT_CTLLO(chan)
734 | DWC_CTLL_DST_WIDTH(dst_width)
735 | DWC_CTLL_SRC_WIDTH(src_width)
736 | DWC_CTLL_DST_INC
737 | DWC_CTLL_SRC_INC
738 | DWC_CTLL_FC_M2M;
739 prev = first = NULL;
740
741 for (offset = 0; offset < len; offset += xfer_count << src_width) {
742 xfer_count = min_t(size_t, (len - offset) >> src_width,
743 dwc->block_size);
744
745 desc = dwc_desc_get(dwc);
746 if (!desc)
747 goto err_desc_get;
748
749 desc->lli.sar = src + offset;
750 desc->lli.dar = dest + offset;
751 desc->lli.ctllo = ctllo;
752 desc->lli.ctlhi = xfer_count;
753
754 if (!first) {
755 first = desc;
756 } else {
757 prev->lli.llp = desc->txd.phys;
758 dma_sync_single_for_device(chan2parent(chan),
759 prev->txd.phys, sizeof(prev->lli),
760 DMA_TO_DEVICE);
761 list_add_tail(&desc->desc_node,
762 &first->tx_list);
763 }
764 prev = desc;
765 }
766
767
768 if (flags & DMA_PREP_INTERRUPT)
769 /* Trigger interrupt after last block */
770 prev->lli.ctllo |= DWC_CTLL_INT_EN;
771
772 prev->lli.llp = 0;
773 dma_sync_single_for_device(chan2parent(chan),
774 prev->txd.phys, sizeof(prev->lli),
775 DMA_TO_DEVICE);
776
777 first->txd.flags = flags;
778 first->len = len;
779
780 return &first->txd;
781
782 err_desc_get:
783 dwc_desc_put(dwc, first);
784 return NULL;
785 }
786
787 static struct dma_async_tx_descriptor *
788 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
789 unsigned int sg_len, enum dma_transfer_direction direction,
790 unsigned long flags, void *context)
791 {
792 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
793 struct dw_dma_slave *dws = chan->private;
794 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
795 struct dw_desc *prev;
796 struct dw_desc *first;
797 u32 ctllo;
798 dma_addr_t reg;
799 unsigned int reg_width;
800 unsigned int mem_width;
801 unsigned int data_width;
802 unsigned int i;
803 struct scatterlist *sg;
804 size_t total_len = 0;
805
806 dev_vdbg(chan2dev(chan), "%s\n", __func__);
807
808 if (unlikely(!is_slave_direction(direction) || !sg_len))
809 return NULL;
810
811 dwc->direction = direction;
812
813 prev = first = NULL;
814
815 switch (direction) {
816 case DMA_MEM_TO_DEV:
817 reg_width = __fls(sconfig->dst_addr_width);
818 reg = sconfig->dst_addr;
819 ctllo = (DWC_DEFAULT_CTLLO(chan)
820 | DWC_CTLL_DST_WIDTH(reg_width)
821 | DWC_CTLL_DST_FIX
822 | DWC_CTLL_SRC_INC);
823
824 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
825 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
826
827 data_width = dwc->dw->data_width[dwc_get_sms(dws)];
828
829 for_each_sg(sgl, sg, sg_len, i) {
830 struct dw_desc *desc;
831 u32 len, dlen, mem;
832
833 mem = sg_dma_address(sg);
834 len = sg_dma_len(sg);
835
836 mem_width = min_t(unsigned int,
837 data_width, dwc_fast_fls(mem | len));
838
839 slave_sg_todev_fill_desc:
840 desc = dwc_desc_get(dwc);
841 if (!desc) {
842 dev_err(chan2dev(chan),
843 "not enough descriptors available\n");
844 goto err_desc_get;
845 }
846
847 desc->lli.sar = mem;
848 desc->lli.dar = reg;
849 desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
850 if ((len >> mem_width) > dwc->block_size) {
851 dlen = dwc->block_size << mem_width;
852 mem += dlen;
853 len -= dlen;
854 } else {
855 dlen = len;
856 len = 0;
857 }
858
859 desc->lli.ctlhi = dlen >> mem_width;
860
861 if (!first) {
862 first = desc;
863 } else {
864 prev->lli.llp = desc->txd.phys;
865 dma_sync_single_for_device(chan2parent(chan),
866 prev->txd.phys,
867 sizeof(prev->lli),
868 DMA_TO_DEVICE);
869 list_add_tail(&desc->desc_node,
870 &first->tx_list);
871 }
872 prev = desc;
873 total_len += dlen;
874
875 if (len)
876 goto slave_sg_todev_fill_desc;
877 }
878 break;
879 case DMA_DEV_TO_MEM:
880 reg_width = __fls(sconfig->src_addr_width);
881 reg = sconfig->src_addr;
882 ctllo = (DWC_DEFAULT_CTLLO(chan)
883 | DWC_CTLL_SRC_WIDTH(reg_width)
884 | DWC_CTLL_DST_INC
885 | DWC_CTLL_SRC_FIX);
886
887 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
888 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
889
890 data_width = dwc->dw->data_width[dwc_get_dms(dws)];
891
892 for_each_sg(sgl, sg, sg_len, i) {
893 struct dw_desc *desc;
894 u32 len, dlen, mem;
895
896 mem = sg_dma_address(sg);
897 len = sg_dma_len(sg);
898
899 mem_width = min_t(unsigned int,
900 data_width, dwc_fast_fls(mem | len));
901
902 slave_sg_fromdev_fill_desc:
903 desc = dwc_desc_get(dwc);
904 if (!desc) {
905 dev_err(chan2dev(chan),
906 "not enough descriptors available\n");
907 goto err_desc_get;
908 }
909
910 desc->lli.sar = reg;
911 desc->lli.dar = mem;
912 desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
913 if ((len >> reg_width) > dwc->block_size) {
914 dlen = dwc->block_size << reg_width;
915 mem += dlen;
916 len -= dlen;
917 } else {
918 dlen = len;
919 len = 0;
920 }
921 desc->lli.ctlhi = dlen >> reg_width;
922
923 if (!first) {
924 first = desc;
925 } else {
926 prev->lli.llp = desc->txd.phys;
927 dma_sync_single_for_device(chan2parent(chan),
928 prev->txd.phys,
929 sizeof(prev->lli),
930 DMA_TO_DEVICE);
931 list_add_tail(&desc->desc_node,
932 &first->tx_list);
933 }
934 prev = desc;
935 total_len += dlen;
936
937 if (len)
938 goto slave_sg_fromdev_fill_desc;
939 }
940 break;
941 default:
942 return NULL;
943 }
944
945 if (flags & DMA_PREP_INTERRUPT)
946 /* Trigger interrupt after last block */
947 prev->lli.ctllo |= DWC_CTLL_INT_EN;
948
949 prev->lli.llp = 0;
950 dma_sync_single_for_device(chan2parent(chan),
951 prev->txd.phys, sizeof(prev->lli),
952 DMA_TO_DEVICE);
953
954 first->len = total_len;
955
956 return &first->txd;
957
958 err_desc_get:
959 dwc_desc_put(dwc, first);
960 return NULL;
961 }
962
963 /*
964 * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
965 * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
966 *
967 * NOTE: burst size 2 is not supported by controller.
968 *
969 * This can be done by finding least significant bit set: n & (n - 1)
970 */
971 static inline void convert_burst(u32 *maxburst)
972 {
973 if (*maxburst > 1)
974 *maxburst = fls(*maxburst) - 2;
975 else
976 *maxburst = 0;
977 }
978
979 static int
980 set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
981 {
982 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
983
984 /* Check if chan will be configured for slave transfers */
985 if (!is_slave_direction(sconfig->direction))
986 return -EINVAL;
987
988 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
989 dwc->direction = sconfig->direction;
990
991 convert_burst(&dwc->dma_sconfig.src_maxburst);
992 convert_burst(&dwc->dma_sconfig.dst_maxburst);
993
994 return 0;
995 }
996
997 static inline void dwc_chan_pause(struct dw_dma_chan *dwc)
998 {
999 u32 cfglo = channel_readl(dwc, CFG_LO);
1000
1001 channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
1002 while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY))
1003 cpu_relax();
1004
1005 dwc->paused = true;
1006 }
1007
1008 static inline void dwc_chan_resume(struct dw_dma_chan *dwc)
1009 {
1010 u32 cfglo = channel_readl(dwc, CFG_LO);
1011
1012 channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
1013
1014 dwc->paused = false;
1015 }
1016
1017 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1018 unsigned long arg)
1019 {
1020 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1021 struct dw_dma *dw = to_dw_dma(chan->device);
1022 struct dw_desc *desc, *_desc;
1023 unsigned long flags;
1024 LIST_HEAD(list);
1025
1026 if (cmd == DMA_PAUSE) {
1027 spin_lock_irqsave(&dwc->lock, flags);
1028
1029 dwc_chan_pause(dwc);
1030
1031 spin_unlock_irqrestore(&dwc->lock, flags);
1032 } else if (cmd == DMA_RESUME) {
1033 if (!dwc->paused)
1034 return 0;
1035
1036 spin_lock_irqsave(&dwc->lock, flags);
1037
1038 dwc_chan_resume(dwc);
1039
1040 spin_unlock_irqrestore(&dwc->lock, flags);
1041 } else if (cmd == DMA_TERMINATE_ALL) {
1042 spin_lock_irqsave(&dwc->lock, flags);
1043
1044 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
1045
1046 dwc_chan_disable(dw, dwc);
1047
1048 dwc->paused = false;
1049
1050 /* active_list entries will end up before queued entries */
1051 list_splice_init(&dwc->queue, &list);
1052 list_splice_init(&dwc->active_list, &list);
1053
1054 spin_unlock_irqrestore(&dwc->lock, flags);
1055
1056 /* Flush all pending and queued descriptors */
1057 list_for_each_entry_safe(desc, _desc, &list, desc_node)
1058 dwc_descriptor_complete(dwc, desc, false);
1059 } else if (cmd == DMA_SLAVE_CONFIG) {
1060 return set_runtime_config(chan, (struct dma_slave_config *)arg);
1061 } else {
1062 return -ENXIO;
1063 }
1064
1065 return 0;
1066 }
1067
1068 static enum dma_status
1069 dwc_tx_status(struct dma_chan *chan,
1070 dma_cookie_t cookie,
1071 struct dma_tx_state *txstate)
1072 {
1073 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1074 enum dma_status ret;
1075
1076 ret = dma_cookie_status(chan, cookie, txstate);
1077 if (ret != DMA_SUCCESS) {
1078 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1079
1080 ret = dma_cookie_status(chan, cookie, txstate);
1081 }
1082
1083 if (ret != DMA_SUCCESS)
1084 dma_set_residue(txstate, dwc_first_active(dwc)->len);
1085
1086 if (dwc->paused)
1087 return DMA_PAUSED;
1088
1089 return ret;
1090 }
1091
1092 static void dwc_issue_pending(struct dma_chan *chan)
1093 {
1094 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1095
1096 if (!list_empty(&dwc->queue))
1097 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1098 }
1099
1100 static int dwc_alloc_chan_resources(struct dma_chan *chan)
1101 {
1102 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1103 struct dw_dma *dw = to_dw_dma(chan->device);
1104 struct dw_desc *desc;
1105 int i;
1106 unsigned long flags;
1107 int ret;
1108
1109 dev_vdbg(chan2dev(chan), "%s\n", __func__);
1110
1111 /* ASSERT: channel is idle */
1112 if (dma_readl(dw, CH_EN) & dwc->mask) {
1113 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
1114 return -EIO;
1115 }
1116
1117 dma_cookie_init(chan);
1118
1119 /*
1120 * NOTE: some controllers may have additional features that we
1121 * need to initialize here, like "scatter-gather" (which
1122 * doesn't mean what you think it means), and status writeback.
1123 */
1124
1125 spin_lock_irqsave(&dwc->lock, flags);
1126 i = dwc->descs_allocated;
1127 while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
1128 spin_unlock_irqrestore(&dwc->lock, flags);
1129
1130 desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
1131 if (!desc)
1132 goto err_desc_alloc;
1133
1134 INIT_LIST_HEAD(&desc->tx_list);
1135 dma_async_tx_descriptor_init(&desc->txd, chan);
1136 desc->txd.tx_submit = dwc_tx_submit;
1137 desc->txd.flags = DMA_CTRL_ACK;
1138 desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
1139 sizeof(desc->lli), DMA_TO_DEVICE);
1140 ret = dma_mapping_error(chan2parent(chan), desc->txd.phys);
1141 if (ret)
1142 goto err_desc_alloc;
1143
1144 dwc_desc_put(dwc, desc);
1145
1146 spin_lock_irqsave(&dwc->lock, flags);
1147 i = ++dwc->descs_allocated;
1148 }
1149
1150 spin_unlock_irqrestore(&dwc->lock, flags);
1151
1152 dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1153
1154 return i;
1155
1156 err_desc_alloc:
1157 kfree(desc);
1158
1159 dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);
1160
1161 return i;
1162 }
1163
1164 static void dwc_free_chan_resources(struct dma_chan *chan)
1165 {
1166 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1167 struct dw_dma *dw = to_dw_dma(chan->device);
1168 struct dw_desc *desc, *_desc;
1169 unsigned long flags;
1170 LIST_HEAD(list);
1171
1172 dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1173 dwc->descs_allocated);
1174
1175 /* ASSERT: channel is idle */
1176 BUG_ON(!list_empty(&dwc->active_list));
1177 BUG_ON(!list_empty(&dwc->queue));
1178 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1179
1180 spin_lock_irqsave(&dwc->lock, flags);
1181 list_splice_init(&dwc->free_list, &list);
1182 dwc->descs_allocated = 0;
1183 dwc->initialized = false;
1184
1185 /* Disable interrupts */
1186 channel_clear_bit(dw, MASK.XFER, dwc->mask);
1187 channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1188
1189 spin_unlock_irqrestore(&dwc->lock, flags);
1190
1191 list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1192 dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
1193 dma_unmap_single(chan2parent(chan), desc->txd.phys,
1194 sizeof(desc->lli), DMA_TO_DEVICE);
1195 kfree(desc);
1196 }
1197
1198 dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1199 }
1200
1201 bool dw_dma_generic_filter(struct dma_chan *chan, void *param)
1202 {
1203 struct dw_dma *dw = to_dw_dma(chan->device);
1204 static struct dw_dma *last_dw;
1205 static char *last_bus_id;
1206 int i = -1;
1207
1208 /*
1209 * dmaengine framework calls this routine for all channels of all dma
1210 * controller, until true is returned. If 'param' bus_id is not
1211 * registered with a dma controller (dw), then there is no need of
1212 * running below function for all channels of dw.
1213 *
1214 * This block of code does this by saving the parameters of last
1215 * failure. If dw and param are same, i.e. trying on same dw with
1216 * different channel, return false.
1217 */
1218 if ((last_dw == dw) && (last_bus_id == param))
1219 return false;
1220 /*
1221 * Return true:
1222 * - If dw_dma's platform data is not filled with slave info, then all
1223 * dma controllers are fine for transfer.
1224 * - Or if param is NULL
1225 */
1226 if (!dw->sd || !param)
1227 return true;
1228
1229 while (++i < dw->sd_count) {
1230 if (!strcmp(dw->sd[i].bus_id, param)) {
1231 chan->private = &dw->sd[i];
1232 last_dw = NULL;
1233 last_bus_id = NULL;
1234
1235 return true;
1236 }
1237 }
1238
1239 last_dw = dw;
1240 last_bus_id = param;
1241 return false;
1242 }
1243 EXPORT_SYMBOL(dw_dma_generic_filter);
1244
1245 /* --------------------- Cyclic DMA API extensions -------------------- */
1246
1247 /**
1248 * dw_dma_cyclic_start - start the cyclic DMA transfer
1249 * @chan: the DMA channel to start
1250 *
1251 * Must be called with soft interrupts disabled. Returns zero on success or
1252 * -errno on failure.
1253 */
1254 int dw_dma_cyclic_start(struct dma_chan *chan)
1255 {
1256 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1257 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1258 unsigned long flags;
1259
1260 if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1261 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1262 return -ENODEV;
1263 }
1264
1265 spin_lock_irqsave(&dwc->lock, flags);
1266
1267 /* assert channel is idle */
1268 if (dma_readl(dw, CH_EN) & dwc->mask) {
1269 dev_err(chan2dev(&dwc->chan),
1270 "BUG: Attempted to start non-idle channel\n");
1271 dwc_dump_chan_regs(dwc);
1272 spin_unlock_irqrestore(&dwc->lock, flags);
1273 return -EBUSY;
1274 }
1275
1276 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1277 dma_writel(dw, CLEAR.XFER, dwc->mask);
1278
1279 /* setup DMAC channel registers */
1280 channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1281 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1282 channel_writel(dwc, CTL_HI, 0);
1283
1284 channel_set_bit(dw, CH_EN, dwc->mask);
1285
1286 spin_unlock_irqrestore(&dwc->lock, flags);
1287
1288 return 0;
1289 }
1290 EXPORT_SYMBOL(dw_dma_cyclic_start);
1291
1292 /**
1293 * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1294 * @chan: the DMA channel to stop
1295 *
1296 * Must be called with soft interrupts disabled.
1297 */
1298 void dw_dma_cyclic_stop(struct dma_chan *chan)
1299 {
1300 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1301 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1302 unsigned long flags;
1303
1304 spin_lock_irqsave(&dwc->lock, flags);
1305
1306 dwc_chan_disable(dw, dwc);
1307
1308 spin_unlock_irqrestore(&dwc->lock, flags);
1309 }
1310 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1311
1312 /**
1313 * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1314 * @chan: the DMA channel to prepare
1315 * @buf_addr: physical DMA address where the buffer starts
1316 * @buf_len: total number of bytes for the entire buffer
1317 * @period_len: number of bytes for each period
1318 * @direction: transfer direction, to or from device
1319 *
1320 * Must be called before trying to start the transfer. Returns a valid struct
1321 * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1322 */
1323 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1324 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1325 enum dma_transfer_direction direction)
1326 {
1327 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1328 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
1329 struct dw_cyclic_desc *cdesc;
1330 struct dw_cyclic_desc *retval = NULL;
1331 struct dw_desc *desc;
1332 struct dw_desc *last = NULL;
1333 unsigned long was_cyclic;
1334 unsigned int reg_width;
1335 unsigned int periods;
1336 unsigned int i;
1337 unsigned long flags;
1338
1339 spin_lock_irqsave(&dwc->lock, flags);
1340 if (dwc->nollp) {
1341 spin_unlock_irqrestore(&dwc->lock, flags);
1342 dev_dbg(chan2dev(&dwc->chan),
1343 "channel doesn't support LLP transfers\n");
1344 return ERR_PTR(-EINVAL);
1345 }
1346
1347 if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1348 spin_unlock_irqrestore(&dwc->lock, flags);
1349 dev_dbg(chan2dev(&dwc->chan),
1350 "queue and/or active list are not empty\n");
1351 return ERR_PTR(-EBUSY);
1352 }
1353
1354 was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1355 spin_unlock_irqrestore(&dwc->lock, flags);
1356 if (was_cyclic) {
1357 dev_dbg(chan2dev(&dwc->chan),
1358 "channel already prepared for cyclic DMA\n");
1359 return ERR_PTR(-EBUSY);
1360 }
1361
1362 retval = ERR_PTR(-EINVAL);
1363
1364 if (unlikely(!is_slave_direction(direction)))
1365 goto out_err;
1366
1367 dwc->direction = direction;
1368
1369 if (direction == DMA_MEM_TO_DEV)
1370 reg_width = __ffs(sconfig->dst_addr_width);
1371 else
1372 reg_width = __ffs(sconfig->src_addr_width);
1373
1374 periods = buf_len / period_len;
1375
1376 /* Check for too big/unaligned periods and unaligned DMA buffer. */
1377 if (period_len > (dwc->block_size << reg_width))
1378 goto out_err;
1379 if (unlikely(period_len & ((1 << reg_width) - 1)))
1380 goto out_err;
1381 if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1382 goto out_err;
1383
1384 retval = ERR_PTR(-ENOMEM);
1385
1386 if (periods > NR_DESCS_PER_CHANNEL)
1387 goto out_err;
1388
1389 cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1390 if (!cdesc)
1391 goto out_err;
1392
1393 cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1394 if (!cdesc->desc)
1395 goto out_err_alloc;
1396
1397 for (i = 0; i < periods; i++) {
1398 desc = dwc_desc_get(dwc);
1399 if (!desc)
1400 goto out_err_desc_get;
1401
1402 switch (direction) {
1403 case DMA_MEM_TO_DEV:
1404 desc->lli.dar = sconfig->dst_addr;
1405 desc->lli.sar = buf_addr + (period_len * i);
1406 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1407 | DWC_CTLL_DST_WIDTH(reg_width)
1408 | DWC_CTLL_SRC_WIDTH(reg_width)
1409 | DWC_CTLL_DST_FIX
1410 | DWC_CTLL_SRC_INC
1411 | DWC_CTLL_INT_EN);
1412
1413 desc->lli.ctllo |= sconfig->device_fc ?
1414 DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
1415 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
1416
1417 break;
1418 case DMA_DEV_TO_MEM:
1419 desc->lli.dar = buf_addr + (period_len * i);
1420 desc->lli.sar = sconfig->src_addr;
1421 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1422 | DWC_CTLL_SRC_WIDTH(reg_width)
1423 | DWC_CTLL_DST_WIDTH(reg_width)
1424 | DWC_CTLL_DST_INC
1425 | DWC_CTLL_SRC_FIX
1426 | DWC_CTLL_INT_EN);
1427
1428 desc->lli.ctllo |= sconfig->device_fc ?
1429 DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
1430 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
1431
1432 break;
1433 default:
1434 break;
1435 }
1436
1437 desc->lli.ctlhi = (period_len >> reg_width);
1438 cdesc->desc[i] = desc;
1439
1440 if (last) {
1441 last->lli.llp = desc->txd.phys;
1442 dma_sync_single_for_device(chan2parent(chan),
1443 last->txd.phys, sizeof(last->lli),
1444 DMA_TO_DEVICE);
1445 }
1446
1447 last = desc;
1448 }
1449
1450 /* lets make a cyclic list */
1451 last->lli.llp = cdesc->desc[0]->txd.phys;
1452 dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
1453 sizeof(last->lli), DMA_TO_DEVICE);
1454
1455 dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
1456 "period %zu periods %d\n", (unsigned long long)buf_addr,
1457 buf_len, period_len, periods);
1458
1459 cdesc->periods = periods;
1460 dwc->cdesc = cdesc;
1461
1462 return cdesc;
1463
1464 out_err_desc_get:
1465 while (i--)
1466 dwc_desc_put(dwc, cdesc->desc[i]);
1467 out_err_alloc:
1468 kfree(cdesc);
1469 out_err:
1470 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1471 return (struct dw_cyclic_desc *)retval;
1472 }
1473 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1474
1475 /**
1476 * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1477 * @chan: the DMA channel to free
1478 */
1479 void dw_dma_cyclic_free(struct dma_chan *chan)
1480 {
1481 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1482 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1483 struct dw_cyclic_desc *cdesc = dwc->cdesc;
1484 int i;
1485 unsigned long flags;
1486
1487 dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
1488
1489 if (!cdesc)
1490 return;
1491
1492 spin_lock_irqsave(&dwc->lock, flags);
1493
1494 dwc_chan_disable(dw, dwc);
1495
1496 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1497 dma_writel(dw, CLEAR.XFER, dwc->mask);
1498
1499 spin_unlock_irqrestore(&dwc->lock, flags);
1500
1501 for (i = 0; i < cdesc->periods; i++)
1502 dwc_desc_put(dwc, cdesc->desc[i]);
1503
1504 kfree(cdesc->desc);
1505 kfree(cdesc);
1506
1507 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1508 }
1509 EXPORT_SYMBOL(dw_dma_cyclic_free);
1510
1511 /*----------------------------------------------------------------------*/
1512
1513 static void dw_dma_off(struct dw_dma *dw)
1514 {
1515 int i;
1516
1517 dma_writel(dw, CFG, 0);
1518
1519 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1520 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1521 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1522 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1523
1524 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1525 cpu_relax();
1526
1527 for (i = 0; i < dw->dma.chancnt; i++)
1528 dw->chan[i].initialized = false;
1529 }
1530
1531 #ifdef CONFIG_OF
1532 static struct dw_dma_platform_data *
1533 dw_dma_parse_dt(struct platform_device *pdev)
1534 {
1535 struct device_node *sn, *cn, *np = pdev->dev.of_node;
1536 struct dw_dma_platform_data *pdata;
1537 struct dw_dma_slave *sd;
1538 u32 tmp, arr[4];
1539
1540 if (!np) {
1541 dev_err(&pdev->dev, "Missing DT data\n");
1542 return NULL;
1543 }
1544
1545 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1546 if (!pdata)
1547 return NULL;
1548
1549 if (of_property_read_u32(np, "nr_channels", &pdata->nr_channels))
1550 return NULL;
1551
1552 if (of_property_read_bool(np, "is_private"))
1553 pdata->is_private = true;
1554
1555 if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
1556 pdata->chan_allocation_order = (unsigned char)tmp;
1557
1558 if (!of_property_read_u32(np, "chan_priority", &tmp))
1559 pdata->chan_priority = tmp;
1560
1561 if (!of_property_read_u32(np, "block_size", &tmp))
1562 pdata->block_size = tmp;
1563
1564 if (!of_property_read_u32(np, "nr_masters", &tmp)) {
1565 if (tmp > 4)
1566 return NULL;
1567
1568 pdata->nr_masters = tmp;
1569 }
1570
1571 if (!of_property_read_u32_array(np, "data_width", arr,
1572 pdata->nr_masters))
1573 for (tmp = 0; tmp < pdata->nr_masters; tmp++)
1574 pdata->data_width[tmp] = arr[tmp];
1575
1576 /* parse slave data */
1577 sn = of_find_node_by_name(np, "slave_info");
1578 if (!sn)
1579 return pdata;
1580
1581 /* calculate number of slaves */
1582 tmp = of_get_child_count(sn);
1583 if (!tmp)
1584 return NULL;
1585
1586 sd = devm_kzalloc(&pdev->dev, sizeof(*sd) * tmp, GFP_KERNEL);
1587 if (!sd)
1588 return NULL;
1589
1590 pdata->sd = sd;
1591 pdata->sd_count = tmp;
1592
1593 for_each_child_of_node(sn, cn) {
1594 sd->dma_dev = &pdev->dev;
1595 of_property_read_string(cn, "bus_id", &sd->bus_id);
1596 of_property_read_u32(cn, "cfg_hi", &sd->cfg_hi);
1597 of_property_read_u32(cn, "cfg_lo", &sd->cfg_lo);
1598 if (!of_property_read_u32(cn, "src_master", &tmp))
1599 sd->src_master = tmp;
1600
1601 if (!of_property_read_u32(cn, "dst_master", &tmp))
1602 sd->dst_master = tmp;
1603 sd++;
1604 }
1605
1606 return pdata;
1607 }
1608 #else
1609 static inline struct dw_dma_platform_data *
1610 dw_dma_parse_dt(struct platform_device *pdev)
1611 {
1612 return NULL;
1613 }
1614 #endif
1615
1616 static int dw_probe(struct platform_device *pdev)
1617 {
1618 struct dw_dma_platform_data *pdata;
1619 struct resource *io;
1620 struct dw_dma *dw;
1621 size_t size;
1622 void __iomem *regs;
1623 bool autocfg;
1624 unsigned int dw_params;
1625 unsigned int nr_channels;
1626 unsigned int max_blk_size = 0;
1627 int irq;
1628 int err;
1629 int i;
1630
1631 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1632 if (!io)
1633 return -EINVAL;
1634
1635 irq = platform_get_irq(pdev, 0);
1636 if (irq < 0)
1637 return irq;
1638
1639 regs = devm_request_and_ioremap(&pdev->dev, io);
1640 if (!regs)
1641 return -EBUSY;
1642
1643 dw_params = dma_read_byaddr(regs, DW_PARAMS);
1644 autocfg = dw_params >> DW_PARAMS_EN & 0x1;
1645
1646 pdata = dev_get_platdata(&pdev->dev);
1647 if (!pdata)
1648 pdata = dw_dma_parse_dt(pdev);
1649
1650 if (!pdata && autocfg) {
1651 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1652 if (!pdata)
1653 return -ENOMEM;
1654
1655 /* Fill platform data with the default values */
1656 pdata->is_private = true;
1657 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1658 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1659 } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1660 return -EINVAL;
1661
1662 if (autocfg)
1663 nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1;
1664 else
1665 nr_channels = pdata->nr_channels;
1666
1667 size = sizeof(struct dw_dma) + nr_channels * sizeof(struct dw_dma_chan);
1668 dw = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1669 if (!dw)
1670 return -ENOMEM;
1671
1672 dw->clk = devm_clk_get(&pdev->dev, "hclk");
1673 if (IS_ERR(dw->clk))
1674 return PTR_ERR(dw->clk);
1675 clk_prepare_enable(dw->clk);
1676
1677 dw->regs = regs;
1678 dw->sd = pdata->sd;
1679 dw->sd_count = pdata->sd_count;
1680
1681 /* get hardware configuration parameters */
1682 if (autocfg) {
1683 max_blk_size = dma_readl(dw, MAX_BLK_SIZE);
1684
1685 dw->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1686 for (i = 0; i < dw->nr_masters; i++) {
1687 dw->data_width[i] =
1688 (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3) + 2;
1689 }
1690 } else {
1691 dw->nr_masters = pdata->nr_masters;
1692 memcpy(dw->data_width, pdata->data_width, 4);
1693 }
1694
1695 /* Calculate all channel mask before DMA setup */
1696 dw->all_chan_mask = (1 << nr_channels) - 1;
1697
1698 /* force dma off, just in case */
1699 dw_dma_off(dw);
1700
1701 /* disable BLOCK interrupts as well */
1702 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1703
1704 err = devm_request_irq(&pdev->dev, irq, dw_dma_interrupt, 0,
1705 "dw_dmac", dw);
1706 if (err)
1707 return err;
1708
1709 platform_set_drvdata(pdev, dw);
1710
1711 tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1712
1713 INIT_LIST_HEAD(&dw->dma.channels);
1714 for (i = 0; i < nr_channels; i++) {
1715 struct dw_dma_chan *dwc = &dw->chan[i];
1716 int r = nr_channels - i - 1;
1717
1718 dwc->chan.device = &dw->dma;
1719 dma_cookie_init(&dwc->chan);
1720 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1721 list_add_tail(&dwc->chan.device_node,
1722 &dw->dma.channels);
1723 else
1724 list_add(&dwc->chan.device_node, &dw->dma.channels);
1725
1726 /* 7 is highest priority & 0 is lowest. */
1727 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1728 dwc->priority = r;
1729 else
1730 dwc->priority = i;
1731
1732 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1733 spin_lock_init(&dwc->lock);
1734 dwc->mask = 1 << i;
1735
1736 INIT_LIST_HEAD(&dwc->active_list);
1737 INIT_LIST_HEAD(&dwc->queue);
1738 INIT_LIST_HEAD(&dwc->free_list);
1739
1740 channel_clear_bit(dw, CH_EN, dwc->mask);
1741
1742 dwc->dw = dw;
1743 dwc->direction = DMA_TRANS_NONE;
1744
1745 /* hardware configuration */
1746 if (autocfg) {
1747 unsigned int dwc_params;
1748
1749 dwc_params = dma_read_byaddr(regs + r * sizeof(u32),
1750 DWC_PARAMS);
1751
1752 /* Decode maximum block size for given channel. The
1753 * stored 4 bit value represents blocks from 0x00 for 3
1754 * up to 0x0a for 4095. */
1755 dwc->block_size =
1756 (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1;
1757 dwc->nollp =
1758 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1759 } else {
1760 dwc->block_size = pdata->block_size;
1761
1762 /* Check if channel supports multi block transfer */
1763 channel_writel(dwc, LLP, 0xfffffffc);
1764 dwc->nollp =
1765 (channel_readl(dwc, LLP) & 0xfffffffc) == 0;
1766 channel_writel(dwc, LLP, 0);
1767 }
1768 }
1769
1770 /* Clear all interrupts on all channels. */
1771 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1772 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1773 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1774 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1775 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1776
1777 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1778 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1779 if (pdata->is_private)
1780 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1781 dw->dma.dev = &pdev->dev;
1782 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1783 dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1784
1785 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1786
1787 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1788 dw->dma.device_control = dwc_control;
1789
1790 dw->dma.device_tx_status = dwc_tx_status;
1791 dw->dma.device_issue_pending = dwc_issue_pending;
1792
1793 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1794
1795 dev_info(&pdev->dev, "DesignWare DMA Controller, %d channels\n",
1796 nr_channels);
1797
1798 dma_async_device_register(&dw->dma);
1799
1800 return 0;
1801 }
1802
1803 static int __devexit dw_remove(struct platform_device *pdev)
1804 {
1805 struct dw_dma *dw = platform_get_drvdata(pdev);
1806 struct dw_dma_chan *dwc, *_dwc;
1807
1808 dw_dma_off(dw);
1809 dma_async_device_unregister(&dw->dma);
1810
1811 tasklet_kill(&dw->tasklet);
1812
1813 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1814 chan.device_node) {
1815 list_del(&dwc->chan.device_node);
1816 channel_clear_bit(dw, CH_EN, dwc->mask);
1817 }
1818
1819 return 0;
1820 }
1821
1822 static void dw_shutdown(struct platform_device *pdev)
1823 {
1824 struct dw_dma *dw = platform_get_drvdata(pdev);
1825
1826 dw_dma_off(dw);
1827 clk_disable_unprepare(dw->clk);
1828 }
1829
1830 static int dw_suspend_noirq(struct device *dev)
1831 {
1832 struct platform_device *pdev = to_platform_device(dev);
1833 struct dw_dma *dw = platform_get_drvdata(pdev);
1834
1835 dw_dma_off(dw);
1836 clk_disable_unprepare(dw->clk);
1837
1838 return 0;
1839 }
1840
1841 static int dw_resume_noirq(struct device *dev)
1842 {
1843 struct platform_device *pdev = to_platform_device(dev);
1844 struct dw_dma *dw = platform_get_drvdata(pdev);
1845
1846 clk_prepare_enable(dw->clk);
1847 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1848
1849 return 0;
1850 }
1851
1852 static const struct dev_pm_ops dw_dev_pm_ops = {
1853 .suspend_noirq = dw_suspend_noirq,
1854 .resume_noirq = dw_resume_noirq,
1855 .freeze_noirq = dw_suspend_noirq,
1856 .thaw_noirq = dw_resume_noirq,
1857 .restore_noirq = dw_resume_noirq,
1858 .poweroff_noirq = dw_suspend_noirq,
1859 };
1860
1861 #ifdef CONFIG_OF
1862 static const struct of_device_id dw_dma_id_table[] = {
1863 { .compatible = "snps,dma-spear1340" },
1864 {}
1865 };
1866 MODULE_DEVICE_TABLE(of, dw_dma_id_table);
1867 #endif
1868
1869 static struct platform_driver dw_driver = {
1870 .probe = dw_probe,
1871 .remove = dw_remove,
1872 .shutdown = dw_shutdown,
1873 .driver = {
1874 .name = "dw_dmac",
1875 .pm = &dw_dev_pm_ops,
1876 .of_match_table = of_match_ptr(dw_dma_id_table),
1877 },
1878 };
1879
1880 static int __init dw_init(void)
1881 {
1882 return platform_driver_register(&dw_driver);
1883 }
1884 subsys_initcall(dw_init);
1885
1886 static void __exit dw_exit(void)
1887 {
1888 platform_driver_unregister(&dw_driver);
1889 }
1890 module_exit(dw_exit);
1891
1892 MODULE_LICENSE("GPL v2");
1893 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1894 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1895 MODULE_AUTHOR("Viresh Kumar <viresh.linux@gmail.com>");
Linux kernel - Deliverables
This page took 0.074927 seconds and 4 git commands to generate.