2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * The full GNU General Public License is included in this distribution in the
15 * file called COPYING.
19 * This code implements the DMA subsystem. It provides a HW-neutral interface
20 * for other kernel code to use asynchronous memory copy capabilities,
21 * if present, and allows different HW DMA drivers to register as providing
24 * Due to the fact we are accelerating what is already a relatively fast
25 * operation, the code goes to great lengths to avoid additional overhead,
30 * The subsystem keeps a global list of dma_device structs it is protected by a
31 * mutex, dma_list_mutex.
33 * A subsystem can get access to a channel by calling dmaengine_get() followed
34 * by dma_find_channel(), or if it has need for an exclusive channel it can call
35 * dma_request_channel(). Once a channel is allocated a reference is taken
36 * against its corresponding driver to disable removal.
38 * Each device has a channels list, which runs unlocked but is never modified
39 * once the device is registered, it's just setup by the driver.
41 * See Documentation/dmaengine.txt for more details
44 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
46 #include <linux/platform_device.h>
47 #include <linux/dma-mapping.h>
48 #include <linux/init.h>
49 #include <linux/module.h>
51 #include <linux/device.h>
52 #include <linux/dmaengine.h>
53 #include <linux/hardirq.h>
54 #include <linux/spinlock.h>
55 #include <linux/percpu.h>
56 #include <linux/rcupdate.h>
57 #include <linux/mutex.h>
58 #include <linux/jiffies.h>
59 #include <linux/rculist.h>
60 #include <linux/idr.h>
61 #include <linux/slab.h>
62 #include <linux/acpi.h>
63 #include <linux/acpi_dma.h>
64 #include <linux/of_dma.h>
65 #include <linux/mempool.h>
67 static DEFINE_MUTEX(dma_list_mutex
);
68 static DEFINE_IDR(dma_idr
);
69 static LIST_HEAD(dma_device_list
);
70 static long dmaengine_ref_count
;
72 /* --- sysfs implementation --- */
75 * dev_to_dma_chan - convert a device pointer to the its sysfs container object
78 * Must be called under dma_list_mutex
80 static struct dma_chan
*dev_to_dma_chan(struct device
*dev
)
82 struct dma_chan_dev
*chan_dev
;
84 chan_dev
= container_of(dev
, typeof(*chan_dev
), device
);
85 return chan_dev
->chan
;
88 static ssize_t
memcpy_count_show(struct device
*dev
,
89 struct device_attribute
*attr
, char *buf
)
91 struct dma_chan
*chan
;
92 unsigned long count
= 0;
96 mutex_lock(&dma_list_mutex
);
97 chan
= dev_to_dma_chan(dev
);
99 for_each_possible_cpu(i
)
100 count
+= per_cpu_ptr(chan
->local
, i
)->memcpy_count
;
101 err
= sprintf(buf
, "%lu\n", count
);
104 mutex_unlock(&dma_list_mutex
);
108 static DEVICE_ATTR_RO(memcpy_count
);
110 static ssize_t
bytes_transferred_show(struct device
*dev
,
111 struct device_attribute
*attr
, char *buf
)
113 struct dma_chan
*chan
;
114 unsigned long count
= 0;
118 mutex_lock(&dma_list_mutex
);
119 chan
= dev_to_dma_chan(dev
);
121 for_each_possible_cpu(i
)
122 count
+= per_cpu_ptr(chan
->local
, i
)->bytes_transferred
;
123 err
= sprintf(buf
, "%lu\n", count
);
126 mutex_unlock(&dma_list_mutex
);
130 static DEVICE_ATTR_RO(bytes_transferred
);
132 static ssize_t
in_use_show(struct device
*dev
, struct device_attribute
*attr
,
135 struct dma_chan
*chan
;
138 mutex_lock(&dma_list_mutex
);
139 chan
= dev_to_dma_chan(dev
);
141 err
= sprintf(buf
, "%d\n", chan
->client_count
);
144 mutex_unlock(&dma_list_mutex
);
148 static DEVICE_ATTR_RO(in_use
);
150 static struct attribute
*dma_dev_attrs
[] = {
151 &dev_attr_memcpy_count
.attr
,
152 &dev_attr_bytes_transferred
.attr
,
153 &dev_attr_in_use
.attr
,
156 ATTRIBUTE_GROUPS(dma_dev
);
158 static void chan_dev_release(struct device
*dev
)
160 struct dma_chan_dev
*chan_dev
;
162 chan_dev
= container_of(dev
, typeof(*chan_dev
), device
);
163 if (atomic_dec_and_test(chan_dev
->idr_ref
)) {
164 mutex_lock(&dma_list_mutex
);
165 idr_remove(&dma_idr
, chan_dev
->dev_id
);
166 mutex_unlock(&dma_list_mutex
);
167 kfree(chan_dev
->idr_ref
);
172 static struct class dma_devclass
= {
174 .dev_groups
= dma_dev_groups
,
175 .dev_release
= chan_dev_release
,
178 /* --- client and device registration --- */
180 #define dma_device_satisfies_mask(device, mask) \
181 __dma_device_satisfies_mask((device), &(mask))
183 __dma_device_satisfies_mask(struct dma_device
*device
,
184 const dma_cap_mask_t
*want
)
188 bitmap_and(has
.bits
, want
->bits
, device
->cap_mask
.bits
,
190 return bitmap_equal(want
->bits
, has
.bits
, DMA_TX_TYPE_END
);
193 static struct module
*dma_chan_to_owner(struct dma_chan
*chan
)
195 return chan
->device
->dev
->driver
->owner
;
199 * balance_ref_count - catch up the channel reference count
200 * @chan - channel to balance ->client_count versus dmaengine_ref_count
202 * balance_ref_count must be called under dma_list_mutex
204 static void balance_ref_count(struct dma_chan
*chan
)
206 struct module
*owner
= dma_chan_to_owner(chan
);
208 while (chan
->client_count
< dmaengine_ref_count
) {
210 chan
->client_count
++;
215 * dma_chan_get - try to grab a dma channel's parent driver module
216 * @chan - channel to grab
218 * Must be called under dma_list_mutex
220 static int dma_chan_get(struct dma_chan
*chan
)
222 struct module
*owner
= dma_chan_to_owner(chan
);
225 /* The channel is already in use, update client count */
226 if (chan
->client_count
) {
231 if (!try_module_get(owner
))
234 /* allocate upon first client reference */
235 if (chan
->device
->device_alloc_chan_resources
) {
236 ret
= chan
->device
->device_alloc_chan_resources(chan
);
241 if (!dma_has_cap(DMA_PRIVATE
, chan
->device
->cap_mask
))
242 balance_ref_count(chan
);
245 chan
->client_count
++;
254 * dma_chan_put - drop a reference to a dma channel's parent driver module
255 * @chan - channel to release
257 * Must be called under dma_list_mutex
259 static void dma_chan_put(struct dma_chan
*chan
)
261 /* This channel is not in use, bail out */
262 if (!chan
->client_count
)
265 chan
->client_count
--;
266 module_put(dma_chan_to_owner(chan
));
268 /* This channel is not in use anymore, free it */
269 if (!chan
->client_count
&& chan
->device
->device_free_chan_resources
)
270 chan
->device
->device_free_chan_resources(chan
);
272 /* If the channel is used via a DMA request router, free the mapping */
273 if (chan
->router
&& chan
->router
->route_free
) {
274 chan
->router
->route_free(chan
->router
->dev
, chan
->route_data
);
276 chan
->route_data
= NULL
;
280 enum dma_status
dma_sync_wait(struct dma_chan
*chan
, dma_cookie_t cookie
)
282 enum dma_status status
;
283 unsigned long dma_sync_wait_timeout
= jiffies
+ msecs_to_jiffies(5000);
285 dma_async_issue_pending(chan
);
287 status
= dma_async_is_tx_complete(chan
, cookie
, NULL
, NULL
);
288 if (time_after_eq(jiffies
, dma_sync_wait_timeout
)) {
289 pr_err("%s: timeout!\n", __func__
);
292 if (status
!= DMA_IN_PROGRESS
)
299 EXPORT_SYMBOL(dma_sync_wait
);
302 * dma_cap_mask_all - enable iteration over all operation types
304 static dma_cap_mask_t dma_cap_mask_all
;
307 * dma_chan_tbl_ent - tracks channel allocations per core/operation
308 * @chan - associated channel for this entry
310 struct dma_chan_tbl_ent
{
311 struct dma_chan
*chan
;
315 * channel_table - percpu lookup table for memory-to-memory offload providers
317 static struct dma_chan_tbl_ent __percpu
*channel_table
[DMA_TX_TYPE_END
];
319 static int __init
dma_channel_table_init(void)
321 enum dma_transaction_type cap
;
324 bitmap_fill(dma_cap_mask_all
.bits
, DMA_TX_TYPE_END
);
326 /* 'interrupt', 'private', and 'slave' are channel capabilities,
327 * but are not associated with an operation so they do not need
328 * an entry in the channel_table
330 clear_bit(DMA_INTERRUPT
, dma_cap_mask_all
.bits
);
331 clear_bit(DMA_PRIVATE
, dma_cap_mask_all
.bits
);
332 clear_bit(DMA_SLAVE
, dma_cap_mask_all
.bits
);
334 for_each_dma_cap_mask(cap
, dma_cap_mask_all
) {
335 channel_table
[cap
] = alloc_percpu(struct dma_chan_tbl_ent
);
336 if (!channel_table
[cap
]) {
343 pr_err("initialization failure\n");
344 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
345 free_percpu(channel_table
[cap
]);
350 arch_initcall(dma_channel_table_init
);
353 * dma_find_channel - find a channel to carry out the operation
354 * @tx_type: transaction type
356 struct dma_chan
*dma_find_channel(enum dma_transaction_type tx_type
)
358 return this_cpu_read(channel_table
[tx_type
]->chan
);
360 EXPORT_SYMBOL(dma_find_channel
);
363 * dma_issue_pending_all - flush all pending operations across all channels
365 void dma_issue_pending_all(void)
367 struct dma_device
*device
;
368 struct dma_chan
*chan
;
371 list_for_each_entry_rcu(device
, &dma_device_list
, global_node
) {
372 if (dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
374 list_for_each_entry(chan
, &device
->channels
, device_node
)
375 if (chan
->client_count
)
376 device
->device_issue_pending(chan
);
380 EXPORT_SYMBOL(dma_issue_pending_all
);
383 * dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu
385 static bool dma_chan_is_local(struct dma_chan
*chan
, int cpu
)
387 int node
= dev_to_node(chan
->device
->dev
);
388 return node
== -1 || cpumask_test_cpu(cpu
, cpumask_of_node(node
));
392 * min_chan - returns the channel with min count and in the same numa-node as the cpu
393 * @cap: capability to match
394 * @cpu: cpu index which the channel should be close to
396 * If some channels are close to the given cpu, the one with the lowest
397 * reference count is returned. Otherwise, cpu is ignored and only the
398 * reference count is taken into account.
399 * Must be called under dma_list_mutex.
401 static struct dma_chan
*min_chan(enum dma_transaction_type cap
, int cpu
)
403 struct dma_device
*device
;
404 struct dma_chan
*chan
;
405 struct dma_chan
*min
= NULL
;
406 struct dma_chan
*localmin
= NULL
;
408 list_for_each_entry(device
, &dma_device_list
, global_node
) {
409 if (!dma_has_cap(cap
, device
->cap_mask
) ||
410 dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
412 list_for_each_entry(chan
, &device
->channels
, device_node
) {
413 if (!chan
->client_count
)
415 if (!min
|| chan
->table_count
< min
->table_count
)
418 if (dma_chan_is_local(chan
, cpu
))
420 chan
->table_count
< localmin
->table_count
)
425 chan
= localmin
? localmin
: min
;
434 * dma_channel_rebalance - redistribute the available channels
436 * Optimize for cpu isolation (each cpu gets a dedicated channel for an
437 * operation type) in the SMP case, and operation isolation (avoid
438 * multi-tasking channels) in the non-SMP case. Must be called under
441 static void dma_channel_rebalance(void)
443 struct dma_chan
*chan
;
444 struct dma_device
*device
;
448 /* undo the last distribution */
449 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
450 for_each_possible_cpu(cpu
)
451 per_cpu_ptr(channel_table
[cap
], cpu
)->chan
= NULL
;
453 list_for_each_entry(device
, &dma_device_list
, global_node
) {
454 if (dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
456 list_for_each_entry(chan
, &device
->channels
, device_node
)
457 chan
->table_count
= 0;
460 /* don't populate the channel_table if no clients are available */
461 if (!dmaengine_ref_count
)
464 /* redistribute available channels */
465 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
466 for_each_online_cpu(cpu
) {
467 chan
= min_chan(cap
, cpu
);
468 per_cpu_ptr(channel_table
[cap
], cpu
)->chan
= chan
;
472 int dma_get_slave_caps(struct dma_chan
*chan
, struct dma_slave_caps
*caps
)
474 struct dma_device
*device
;
479 device
= chan
->device
;
481 /* check if the channel supports slave transactions */
482 if (!test_bit(DMA_SLAVE
, device
->cap_mask
.bits
))
486 * Check whether it reports it uses the generic slave
487 * capabilities, if not, that means it doesn't support any
488 * kind of slave capabilities reporting.
490 if (!device
->directions
)
493 caps
->src_addr_widths
= device
->src_addr_widths
;
494 caps
->dst_addr_widths
= device
->dst_addr_widths
;
495 caps
->directions
= device
->directions
;
496 caps
->residue_granularity
= device
->residue_granularity
;
497 caps
->descriptor_reuse
= device
->descriptor_reuse
;
500 * Some devices implement only pause (e.g. to get residuum) but no
501 * resume. However cmd_pause is advertised as pause AND resume.
503 caps
->cmd_pause
= !!(device
->device_pause
&& device
->device_resume
);
504 caps
->cmd_terminate
= !!device
->device_terminate_all
;
508 EXPORT_SYMBOL_GPL(dma_get_slave_caps
);
510 static struct dma_chan
*private_candidate(const dma_cap_mask_t
*mask
,
511 struct dma_device
*dev
,
512 dma_filter_fn fn
, void *fn_param
)
514 struct dma_chan
*chan
;
516 if (mask
&& !__dma_device_satisfies_mask(dev
, mask
)) {
517 pr_debug("%s: wrong capabilities\n", __func__
);
520 /* devices with multiple channels need special handling as we need to
521 * ensure that all channels are either private or public.
523 if (dev
->chancnt
> 1 && !dma_has_cap(DMA_PRIVATE
, dev
->cap_mask
))
524 list_for_each_entry(chan
, &dev
->channels
, device_node
) {
525 /* some channels are already publicly allocated */
526 if (chan
->client_count
)
530 list_for_each_entry(chan
, &dev
->channels
, device_node
) {
531 if (chan
->client_count
) {
532 pr_debug("%s: %s busy\n",
533 __func__
, dma_chan_name(chan
));
536 if (fn
&& !fn(chan
, fn_param
)) {
537 pr_debug("%s: %s filter said false\n",
538 __func__
, dma_chan_name(chan
));
547 static struct dma_chan
*find_candidate(struct dma_device
*device
,
548 const dma_cap_mask_t
*mask
,
549 dma_filter_fn fn
, void *fn_param
)
551 struct dma_chan
*chan
= private_candidate(mask
, device
, fn
, fn_param
);
555 /* Found a suitable channel, try to grab, prep, and return it.
556 * We first set DMA_PRIVATE to disable balance_ref_count as this
557 * channel will not be published in the general-purpose
560 dma_cap_set(DMA_PRIVATE
, device
->cap_mask
);
561 device
->privatecnt
++;
562 err
= dma_chan_get(chan
);
565 if (err
== -ENODEV
) {
566 pr_debug("%s: %s module removed\n", __func__
,
567 dma_chan_name(chan
));
568 list_del_rcu(&device
->global_node
);
570 pr_debug("%s: failed to get %s: (%d)\n",
571 __func__
, dma_chan_name(chan
), err
);
573 if (--device
->privatecnt
== 0)
574 dma_cap_clear(DMA_PRIVATE
, device
->cap_mask
);
580 return chan
? chan
: ERR_PTR(-EPROBE_DEFER
);
584 * dma_get_slave_channel - try to get specific channel exclusively
585 * @chan: target channel
587 struct dma_chan
*dma_get_slave_channel(struct dma_chan
*chan
)
591 /* lock against __dma_request_channel */
592 mutex_lock(&dma_list_mutex
);
594 if (chan
->client_count
== 0) {
595 struct dma_device
*device
= chan
->device
;
597 dma_cap_set(DMA_PRIVATE
, device
->cap_mask
);
598 device
->privatecnt
++;
599 err
= dma_chan_get(chan
);
601 pr_debug("%s: failed to get %s: (%d)\n",
602 __func__
, dma_chan_name(chan
), err
);
604 if (--device
->privatecnt
== 0)
605 dma_cap_clear(DMA_PRIVATE
, device
->cap_mask
);
610 mutex_unlock(&dma_list_mutex
);
615 EXPORT_SYMBOL_GPL(dma_get_slave_channel
);
617 struct dma_chan
*dma_get_any_slave_channel(struct dma_device
*device
)
620 struct dma_chan
*chan
;
623 dma_cap_set(DMA_SLAVE
, mask
);
625 /* lock against __dma_request_channel */
626 mutex_lock(&dma_list_mutex
);
628 chan
= find_candidate(device
, &mask
, NULL
, NULL
);
630 mutex_unlock(&dma_list_mutex
);
632 return IS_ERR(chan
) ? NULL
: chan
;
634 EXPORT_SYMBOL_GPL(dma_get_any_slave_channel
);
637 * __dma_request_channel - try to allocate an exclusive channel
638 * @mask: capabilities that the channel must satisfy
639 * @fn: optional callback to disposition available channels
640 * @fn_param: opaque parameter to pass to dma_filter_fn
642 * Returns pointer to appropriate DMA channel on success or NULL.
644 struct dma_chan
*__dma_request_channel(const dma_cap_mask_t
*mask
,
645 dma_filter_fn fn
, void *fn_param
)
647 struct dma_device
*device
, *_d
;
648 struct dma_chan
*chan
= NULL
;
651 mutex_lock(&dma_list_mutex
);
652 list_for_each_entry_safe(device
, _d
, &dma_device_list
, global_node
) {
653 chan
= find_candidate(device
, mask
, fn
, fn_param
);
659 mutex_unlock(&dma_list_mutex
);
661 pr_debug("%s: %s (%s)\n",
663 chan
? "success" : "fail",
664 chan
? dma_chan_name(chan
) : NULL
);
668 EXPORT_SYMBOL_GPL(__dma_request_channel
);
670 static const struct dma_slave_map
*dma_filter_match(struct dma_device
*device
,
676 if (!device
->filter
.mapcnt
)
679 for (i
= 0; i
< device
->filter
.mapcnt
; i
++) {
680 const struct dma_slave_map
*map
= &device
->filter
.map
[i
];
682 if (!strcmp(map
->devname
, dev_name(dev
)) &&
683 !strcmp(map
->slave
, name
))
691 * dma_request_chan - try to allocate an exclusive slave channel
692 * @dev: pointer to client device structure
693 * @name: slave channel name
695 * Returns pointer to appropriate DMA channel on success or an error pointer.
697 struct dma_chan
*dma_request_chan(struct device
*dev
, const char *name
)
699 struct dma_device
*d
, *_d
;
700 struct dma_chan
*chan
= NULL
;
702 /* If device-tree is present get slave info from here */
704 chan
= of_dma_request_slave_channel(dev
->of_node
, name
);
706 /* If device was enumerated by ACPI get slave info from here */
707 if (has_acpi_companion(dev
) && !chan
)
708 chan
= acpi_dma_request_slave_chan_by_name(dev
, name
);
711 /* Valid channel found or requester need to be deferred */
712 if (!IS_ERR(chan
) || PTR_ERR(chan
) == -EPROBE_DEFER
)
716 /* Try to find the channel via the DMA filter map(s) */
717 mutex_lock(&dma_list_mutex
);
718 list_for_each_entry_safe(d
, _d
, &dma_device_list
, global_node
) {
720 const struct dma_slave_map
*map
= dma_filter_match(d
, name
, dev
);
726 dma_cap_set(DMA_SLAVE
, mask
);
728 chan
= find_candidate(d
, &mask
, d
->filter
.fn
, map
->param
);
732 mutex_unlock(&dma_list_mutex
);
734 return chan
? chan
: ERR_PTR(-EPROBE_DEFER
);
736 EXPORT_SYMBOL_GPL(dma_request_chan
);
739 * dma_request_slave_channel - try to allocate an exclusive slave channel
740 * @dev: pointer to client device structure
741 * @name: slave channel name
743 * Returns pointer to appropriate DMA channel on success or NULL.
745 struct dma_chan
*dma_request_slave_channel(struct device
*dev
,
748 struct dma_chan
*ch
= dma_request_chan(dev
, name
);
754 EXPORT_SYMBOL_GPL(dma_request_slave_channel
);
757 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
758 * @mask: capabilities that the channel must satisfy
760 * Returns pointer to appropriate DMA channel on success or an error pointer.
762 struct dma_chan
*dma_request_chan_by_mask(const dma_cap_mask_t
*mask
)
764 struct dma_chan
*chan
;
767 return ERR_PTR(-ENODEV
);
769 chan
= __dma_request_channel(mask
, NULL
, NULL
);
771 chan
= ERR_PTR(-ENODEV
);
775 EXPORT_SYMBOL_GPL(dma_request_chan_by_mask
);
777 void dma_release_channel(struct dma_chan
*chan
)
779 mutex_lock(&dma_list_mutex
);
780 WARN_ONCE(chan
->client_count
!= 1,
781 "chan reference count %d != 1\n", chan
->client_count
);
783 /* drop PRIVATE cap enabled by __dma_request_channel() */
784 if (--chan
->device
->privatecnt
== 0)
785 dma_cap_clear(DMA_PRIVATE
, chan
->device
->cap_mask
);
786 mutex_unlock(&dma_list_mutex
);
788 EXPORT_SYMBOL_GPL(dma_release_channel
);
791 * dmaengine_get - register interest in dma_channels
793 void dmaengine_get(void)
795 struct dma_device
*device
, *_d
;
796 struct dma_chan
*chan
;
799 mutex_lock(&dma_list_mutex
);
800 dmaengine_ref_count
++;
802 /* try to grab channels */
803 list_for_each_entry_safe(device
, _d
, &dma_device_list
, global_node
) {
804 if (dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
806 list_for_each_entry(chan
, &device
->channels
, device_node
) {
807 err
= dma_chan_get(chan
);
808 if (err
== -ENODEV
) {
809 /* module removed before we could use it */
810 list_del_rcu(&device
->global_node
);
813 pr_debug("%s: failed to get %s: (%d)\n",
814 __func__
, dma_chan_name(chan
), err
);
818 /* if this is the first reference and there were channels
819 * waiting we need to rebalance to get those channels
820 * incorporated into the channel table
822 if (dmaengine_ref_count
== 1)
823 dma_channel_rebalance();
824 mutex_unlock(&dma_list_mutex
);
826 EXPORT_SYMBOL(dmaengine_get
);
829 * dmaengine_put - let dma drivers be removed when ref_count == 0
831 void dmaengine_put(void)
833 struct dma_device
*device
;
834 struct dma_chan
*chan
;
836 mutex_lock(&dma_list_mutex
);
837 dmaengine_ref_count
--;
838 BUG_ON(dmaengine_ref_count
< 0);
839 /* drop channel references */
840 list_for_each_entry(device
, &dma_device_list
, global_node
) {
841 if (dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
843 list_for_each_entry(chan
, &device
->channels
, device_node
)
846 mutex_unlock(&dma_list_mutex
);
848 EXPORT_SYMBOL(dmaengine_put
);
850 static bool device_has_all_tx_types(struct dma_device
*device
)
852 /* A device that satisfies this test has channels that will never cause
853 * an async_tx channel switch event as all possible operation types can
856 #ifdef CONFIG_ASYNC_TX_DMA
857 if (!dma_has_cap(DMA_INTERRUPT
, device
->cap_mask
))
861 #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
862 if (!dma_has_cap(DMA_MEMCPY
, device
->cap_mask
))
866 #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
867 if (!dma_has_cap(DMA_XOR
, device
->cap_mask
))
870 #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
871 if (!dma_has_cap(DMA_XOR_VAL
, device
->cap_mask
))
876 #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
877 if (!dma_has_cap(DMA_PQ
, device
->cap_mask
))
880 #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
881 if (!dma_has_cap(DMA_PQ_VAL
, device
->cap_mask
))
889 static int get_dma_id(struct dma_device
*device
)
893 mutex_lock(&dma_list_mutex
);
895 rc
= idr_alloc(&dma_idr
, NULL
, 0, 0, GFP_KERNEL
);
899 mutex_unlock(&dma_list_mutex
);
900 return rc
< 0 ? rc
: 0;
904 * dma_async_device_register - registers DMA devices found
905 * @device: &dma_device
907 int dma_async_device_register(struct dma_device
*device
)
910 struct dma_chan
* chan
;
916 /* validate device routines */
917 BUG_ON(dma_has_cap(DMA_MEMCPY
, device
->cap_mask
) &&
918 !device
->device_prep_dma_memcpy
);
919 BUG_ON(dma_has_cap(DMA_XOR
, device
->cap_mask
) &&
920 !device
->device_prep_dma_xor
);
921 BUG_ON(dma_has_cap(DMA_XOR_VAL
, device
->cap_mask
) &&
922 !device
->device_prep_dma_xor_val
);
923 BUG_ON(dma_has_cap(DMA_PQ
, device
->cap_mask
) &&
924 !device
->device_prep_dma_pq
);
925 BUG_ON(dma_has_cap(DMA_PQ_VAL
, device
->cap_mask
) &&
926 !device
->device_prep_dma_pq_val
);
927 BUG_ON(dma_has_cap(DMA_MEMSET
, device
->cap_mask
) &&
928 !device
->device_prep_dma_memset
);
929 BUG_ON(dma_has_cap(DMA_INTERRUPT
, device
->cap_mask
) &&
930 !device
->device_prep_dma_interrupt
);
931 BUG_ON(dma_has_cap(DMA_SG
, device
->cap_mask
) &&
932 !device
->device_prep_dma_sg
);
933 BUG_ON(dma_has_cap(DMA_CYCLIC
, device
->cap_mask
) &&
934 !device
->device_prep_dma_cyclic
);
935 BUG_ON(dma_has_cap(DMA_INTERLEAVE
, device
->cap_mask
) &&
936 !device
->device_prep_interleaved_dma
);
938 BUG_ON(!device
->device_tx_status
);
939 BUG_ON(!device
->device_issue_pending
);
940 BUG_ON(!device
->dev
);
942 /* note: this only matters in the
943 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
945 if (device_has_all_tx_types(device
))
946 dma_cap_set(DMA_ASYNC_TX
, device
->cap_mask
);
948 idr_ref
= kmalloc(sizeof(*idr_ref
), GFP_KERNEL
);
951 rc
= get_dma_id(device
);
957 atomic_set(idr_ref
, 0);
959 /* represent channels in sysfs. Probably want devs too */
960 list_for_each_entry(chan
, &device
->channels
, device_node
) {
962 chan
->local
= alloc_percpu(typeof(*chan
->local
));
963 if (chan
->local
== NULL
)
965 chan
->dev
= kzalloc(sizeof(*chan
->dev
), GFP_KERNEL
);
966 if (chan
->dev
== NULL
) {
967 free_percpu(chan
->local
);
972 chan
->chan_id
= chancnt
++;
973 chan
->dev
->device
.class = &dma_devclass
;
974 chan
->dev
->device
.parent
= device
->dev
;
975 chan
->dev
->chan
= chan
;
976 chan
->dev
->idr_ref
= idr_ref
;
977 chan
->dev
->dev_id
= device
->dev_id
;
979 dev_set_name(&chan
->dev
->device
, "dma%dchan%d",
980 device
->dev_id
, chan
->chan_id
);
982 rc
= device_register(&chan
->dev
->device
);
984 free_percpu(chan
->local
);
990 chan
->client_count
= 0;
992 device
->chancnt
= chancnt
;
994 mutex_lock(&dma_list_mutex
);
995 /* take references on public channels */
996 if (dmaengine_ref_count
&& !dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
997 list_for_each_entry(chan
, &device
->channels
, device_node
) {
998 /* if clients are already waiting for channels we need
999 * to take references on their behalf
1001 if (dma_chan_get(chan
) == -ENODEV
) {
1002 /* note we can only get here for the first
1003 * channel as the remaining channels are
1004 * guaranteed to get a reference
1007 mutex_unlock(&dma_list_mutex
);
1011 list_add_tail_rcu(&device
->global_node
, &dma_device_list
);
1012 if (dma_has_cap(DMA_PRIVATE
, device
->cap_mask
))
1013 device
->privatecnt
++; /* Always private */
1014 dma_channel_rebalance();
1015 mutex_unlock(&dma_list_mutex
);
1020 /* if we never registered a channel just release the idr */
1021 if (atomic_read(idr_ref
) == 0) {
1022 mutex_lock(&dma_list_mutex
);
1023 idr_remove(&dma_idr
, device
->dev_id
);
1024 mutex_unlock(&dma_list_mutex
);
1029 list_for_each_entry(chan
, &device
->channels
, device_node
) {
1030 if (chan
->local
== NULL
)
1032 mutex_lock(&dma_list_mutex
);
1033 chan
->dev
->chan
= NULL
;
1034 mutex_unlock(&dma_list_mutex
);
1035 device_unregister(&chan
->dev
->device
);
1036 free_percpu(chan
->local
);
1040 EXPORT_SYMBOL(dma_async_device_register
);
1043 * dma_async_device_unregister - unregister a DMA device
1044 * @device: &dma_device
1046 * This routine is called by dma driver exit routines, dmaengine holds module
1047 * references to prevent it being called while channels are in use.
1049 void dma_async_device_unregister(struct dma_device
*device
)
1051 struct dma_chan
*chan
;
1053 mutex_lock(&dma_list_mutex
);
1054 list_del_rcu(&device
->global_node
);
1055 dma_channel_rebalance();
1056 mutex_unlock(&dma_list_mutex
);
1058 list_for_each_entry(chan
, &device
->channels
, device_node
) {
1059 WARN_ONCE(chan
->client_count
,
1060 "%s called while %d clients hold a reference\n",
1061 __func__
, chan
->client_count
);
1062 mutex_lock(&dma_list_mutex
);
1063 chan
->dev
->chan
= NULL
;
1064 mutex_unlock(&dma_list_mutex
);
1065 device_unregister(&chan
->dev
->device
);
1066 free_percpu(chan
->local
);
1069 EXPORT_SYMBOL(dma_async_device_unregister
);
1071 struct dmaengine_unmap_pool
{
1072 struct kmem_cache
*cache
;
1078 #define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
1079 static struct dmaengine_unmap_pool unmap_pool
[] = {
1081 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1088 static struct dmaengine_unmap_pool
*__get_unmap_pool(int nr
)
1090 int order
= get_count_order(nr
);
1094 return &unmap_pool
[0];
1096 return &unmap_pool
[1];
1098 return &unmap_pool
[2];
1100 return &unmap_pool
[3];
1107 static void dmaengine_unmap(struct kref
*kref
)
1109 struct dmaengine_unmap_data
*unmap
= container_of(kref
, typeof(*unmap
), kref
);
1110 struct device
*dev
= unmap
->dev
;
1113 cnt
= unmap
->to_cnt
;
1114 for (i
= 0; i
< cnt
; i
++)
1115 dma_unmap_page(dev
, unmap
->addr
[i
], unmap
->len
,
1117 cnt
+= unmap
->from_cnt
;
1118 for (; i
< cnt
; i
++)
1119 dma_unmap_page(dev
, unmap
->addr
[i
], unmap
->len
,
1121 cnt
+= unmap
->bidi_cnt
;
1122 for (; i
< cnt
; i
++) {
1123 if (unmap
->addr
[i
] == 0)
1125 dma_unmap_page(dev
, unmap
->addr
[i
], unmap
->len
,
1128 cnt
= unmap
->map_cnt
;
1129 mempool_free(unmap
, __get_unmap_pool(cnt
)->pool
);
1132 void dmaengine_unmap_put(struct dmaengine_unmap_data
*unmap
)
1135 kref_put(&unmap
->kref
, dmaengine_unmap
);
1137 EXPORT_SYMBOL_GPL(dmaengine_unmap_put
);
1139 static void dmaengine_destroy_unmap_pool(void)
1143 for (i
= 0; i
< ARRAY_SIZE(unmap_pool
); i
++) {
1144 struct dmaengine_unmap_pool
*p
= &unmap_pool
[i
];
1146 mempool_destroy(p
->pool
);
1148 kmem_cache_destroy(p
->cache
);
1153 static int __init
dmaengine_init_unmap_pool(void)
1157 for (i
= 0; i
< ARRAY_SIZE(unmap_pool
); i
++) {
1158 struct dmaengine_unmap_pool
*p
= &unmap_pool
[i
];
1161 size
= sizeof(struct dmaengine_unmap_data
) +
1162 sizeof(dma_addr_t
) * p
->size
;
1164 p
->cache
= kmem_cache_create(p
->name
, size
, 0,
1165 SLAB_HWCACHE_ALIGN
, NULL
);
1168 p
->pool
= mempool_create_slab_pool(1, p
->cache
);
1173 if (i
== ARRAY_SIZE(unmap_pool
))
1176 dmaengine_destroy_unmap_pool();
1180 struct dmaengine_unmap_data
*
1181 dmaengine_get_unmap_data(struct device
*dev
, int nr
, gfp_t flags
)
1183 struct dmaengine_unmap_data
*unmap
;
1185 unmap
= mempool_alloc(__get_unmap_pool(nr
)->pool
, flags
);
1189 memset(unmap
, 0, sizeof(*unmap
));
1190 kref_init(&unmap
->kref
);
1192 unmap
->map_cnt
= nr
;
1196 EXPORT_SYMBOL(dmaengine_get_unmap_data
);
1198 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor
*tx
,
1199 struct dma_chan
*chan
)
1202 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1203 spin_lock_init(&tx
->lock
);
1206 EXPORT_SYMBOL(dma_async_tx_descriptor_init
);
1208 /* dma_wait_for_async_tx - spin wait for a transaction to complete
1209 * @tx: in-flight transaction to wait on
1212 dma_wait_for_async_tx(struct dma_async_tx_descriptor
*tx
)
1214 unsigned long dma_sync_wait_timeout
= jiffies
+ msecs_to_jiffies(5000);
1217 return DMA_COMPLETE
;
1219 while (tx
->cookie
== -EBUSY
) {
1220 if (time_after_eq(jiffies
, dma_sync_wait_timeout
)) {
1221 pr_err("%s timeout waiting for descriptor submission\n",
1227 return dma_sync_wait(tx
->chan
, tx
->cookie
);
1229 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx
);
1231 /* dma_run_dependencies - helper routine for dma drivers to process
1232 * (start) dependent operations on their target channel
1233 * @tx: transaction with dependencies
1235 void dma_run_dependencies(struct dma_async_tx_descriptor
*tx
)
1237 struct dma_async_tx_descriptor
*dep
= txd_next(tx
);
1238 struct dma_async_tx_descriptor
*dep_next
;
1239 struct dma_chan
*chan
;
1244 /* we'll submit tx->next now, so clear the link */
1248 /* keep submitting up until a channel switch is detected
1249 * in that case we will be called again as a result of
1250 * processing the interrupt from async_tx_channel_switch
1252 for (; dep
; dep
= dep_next
) {
1254 txd_clear_parent(dep
);
1255 dep_next
= txd_next(dep
);
1256 if (dep_next
&& dep_next
->chan
== chan
)
1257 txd_clear_next(dep
); /* ->next will be submitted */
1259 dep_next
= NULL
; /* submit current dep and terminate */
1262 dep
->tx_submit(dep
);
1265 chan
->device
->device_issue_pending(chan
);
1267 EXPORT_SYMBOL_GPL(dma_run_dependencies
);
1269 static int __init
dma_bus_init(void)
1271 int err
= dmaengine_init_unmap_pool();
1275 return class_register(&dma_devclass
);
1277 arch_initcall(dma_bus_init
);