2 * core routines for the asynchronous memory transfer/transform api
4 * Copyright © 2006, Intel Corporation.
6 * Dan Williams <dan.j.williams@intel.com>
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/async_tx.h>
29 #ifdef CONFIG_DMA_ENGINE
30 static enum dma_state_client
31 dma_channel_add_remove(struct dma_client
*client
,
32 struct dma_chan
*chan
, enum dma_state state
);
34 static struct dma_client async_tx_dma
= {
35 .event_callback
= dma_channel_add_remove
,
36 /* .cap_mask == 0 defaults to all channels */
40 * dma_cap_mask_all - enable iteration over all operation types
42 static dma_cap_mask_t dma_cap_mask_all
;
45 * chan_ref_percpu - tracks channel allocations per core/opertion
47 struct chan_ref_percpu
{
48 struct dma_chan_ref
*ref
;
51 static int channel_table_initialized
;
52 static struct chan_ref_percpu
*channel_table
[DMA_TX_TYPE_END
];
55 * async_tx_lock - protect modification of async_tx_master_list and serialize
56 * rebalance operations
58 static spinlock_t async_tx_lock
;
60 static struct list_head
61 async_tx_master_list
= LIST_HEAD_INIT(async_tx_master_list
);
63 /* async_tx_issue_pending_all - start all transactions on all channels */
64 void async_tx_issue_pending_all(void)
66 struct dma_chan_ref
*ref
;
69 list_for_each_entry_rcu(ref
, &async_tx_master_list
, node
)
70 ref
->chan
->device
->device_issue_pending(ref
->chan
);
73 EXPORT_SYMBOL_GPL(async_tx_issue_pending_all
);
75 /* dma_wait_for_async_tx - spin wait for a transcation to complete
76 * @tx: transaction to wait on
79 dma_wait_for_async_tx(struct dma_async_tx_descriptor
*tx
)
81 enum dma_status status
;
82 struct dma_async_tx_descriptor
*iter
;
87 /* poll through the dependency chain, return when tx is complete */
90 while (iter
->cookie
== -EBUSY
)
93 status
= dma_sync_wait(iter
->chan
, iter
->cookie
);
94 } while (status
== DMA_IN_PROGRESS
|| (iter
!= tx
));
98 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx
);
100 /* async_tx_run_dependencies - helper routine for dma drivers to process
101 * (start) dependent operations on their target channel
102 * @tx: transaction with dependencies
105 async_tx_run_dependencies(struct dma_async_tx_descriptor
*tx
)
107 struct dma_async_tx_descriptor
*dep_tx
, *_dep_tx
;
108 struct dma_device
*dev
;
109 struct dma_chan
*chan
;
111 list_for_each_entry_safe(dep_tx
, _dep_tx
, &tx
->depend_list
,
115 /* we can't depend on ourselves */
116 BUG_ON(chan
== tx
->chan
);
117 list_del(&dep_tx
->depend_node
);
118 tx
->tx_submit(dep_tx
);
120 /* we need to poke the engine as client code does not
121 * know about dependency submission events
123 dev
->device_issue_pending(chan
);
126 EXPORT_SYMBOL_GPL(async_tx_run_dependencies
);
129 free_dma_chan_ref(struct rcu_head
*rcu
)
131 struct dma_chan_ref
*ref
;
132 ref
= container_of(rcu
, struct dma_chan_ref
, rcu
);
137 init_dma_chan_ref(struct dma_chan_ref
*ref
, struct dma_chan
*chan
)
139 INIT_LIST_HEAD(&ref
->node
);
140 INIT_RCU_HEAD(&ref
->rcu
);
142 atomic_set(&ref
->count
, 0);
146 * get_chan_ref_by_cap - returns the nth channel of the given capability
147 * defaults to returning the channel with the desired capability and the
148 * lowest reference count if the index can not be satisfied
149 * @cap: capability to match
150 * @index: nth channel desired, passing -1 has the effect of forcing the
151 * default return value
153 static struct dma_chan_ref
*
154 get_chan_ref_by_cap(enum dma_transaction_type cap
, int index
)
156 struct dma_chan_ref
*ret_ref
= NULL
, *min_ref
= NULL
, *ref
;
159 list_for_each_entry_rcu(ref
, &async_tx_master_list
, node
)
160 if (dma_has_cap(cap
, ref
->chan
->device
->cap_mask
)) {
163 else if (atomic_read(&ref
->count
) <
164 atomic_read(&min_ref
->count
))
178 atomic_inc(&ret_ref
->count
);
184 * async_tx_rebalance - redistribute the available channels, optimize
185 * for cpu isolation in the SMP case, and opertaion isolation in the
188 static void async_tx_rebalance(void)
190 int cpu
, cap
, cpu_idx
= 0;
193 if (!channel_table_initialized
)
196 spin_lock_irqsave(&async_tx_lock
, flags
);
198 /* undo the last distribution */
199 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
200 for_each_possible_cpu(cpu
) {
201 struct dma_chan_ref
*ref
=
202 per_cpu_ptr(channel_table
[cap
], cpu
)->ref
;
204 atomic_set(&ref
->count
, 0);
205 per_cpu_ptr(channel_table
[cap
], cpu
)->ref
=
210 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
211 for_each_online_cpu(cpu
) {
212 struct dma_chan_ref
*new;
214 new = get_chan_ref_by_cap(cap
, cpu_idx
++);
216 new = get_chan_ref_by_cap(cap
, -1);
218 per_cpu_ptr(channel_table
[cap
], cpu
)->ref
= new;
221 spin_unlock_irqrestore(&async_tx_lock
, flags
);
224 static enum dma_state_client
225 dma_channel_add_remove(struct dma_client
*client
,
226 struct dma_chan
*chan
, enum dma_state state
)
228 unsigned long found
, flags
;
229 struct dma_chan_ref
*master_ref
, *ref
;
230 enum dma_state_client ack
= DMA_DUP
; /* default: take no action */
233 case DMA_RESOURCE_AVAILABLE
:
236 list_for_each_entry_rcu(ref
, &async_tx_master_list
, node
)
237 if (ref
->chan
== chan
) {
243 pr_debug("async_tx: dma resource available [%s]\n",
244 found
? "old" : "new");
251 /* add the channel to the generic management list */
252 master_ref
= kmalloc(sizeof(*master_ref
), GFP_KERNEL
);
254 /* keep a reference until async_tx is unloaded */
256 init_dma_chan_ref(master_ref
, chan
);
257 spin_lock_irqsave(&async_tx_lock
, flags
);
258 list_add_tail_rcu(&master_ref
->node
,
259 &async_tx_master_list
);
260 spin_unlock_irqrestore(&async_tx_lock
,
263 printk(KERN_WARNING
"async_tx: unable to create"
264 " new master entry in response to"
265 " a DMA_RESOURCE_ADDED event"
270 async_tx_rebalance();
272 case DMA_RESOURCE_REMOVED
:
274 spin_lock_irqsave(&async_tx_lock
, flags
);
275 list_for_each_entry_rcu(ref
, &async_tx_master_list
, node
)
276 if (ref
->chan
== chan
) {
277 /* permit backing devices to go away */
278 dma_chan_put(ref
->chan
);
279 list_del_rcu(&ref
->node
);
280 call_rcu(&ref
->rcu
, free_dma_chan_ref
);
284 spin_unlock_irqrestore(&async_tx_lock
, flags
);
286 pr_debug("async_tx: dma resource removed [%s]\n",
287 found
? "ours" : "not ours");
294 async_tx_rebalance();
296 case DMA_RESOURCE_SUSPEND
:
297 case DMA_RESOURCE_RESUME
:
298 printk(KERN_WARNING
"async_tx: does not support dma channel"
299 " suspend/resume\n");
311 enum dma_transaction_type cap
;
313 spin_lock_init(&async_tx_lock
);
314 bitmap_fill(dma_cap_mask_all
.bits
, DMA_TX_TYPE_END
);
316 /* an interrupt will never be an explicit operation type.
317 * clearing this bit prevents allocation to a slot in 'channel_table'
319 clear_bit(DMA_INTERRUPT
, dma_cap_mask_all
.bits
);
321 for_each_dma_cap_mask(cap
, dma_cap_mask_all
) {
322 channel_table
[cap
] = alloc_percpu(struct chan_ref_percpu
);
323 if (!channel_table
[cap
])
327 channel_table_initialized
= 1;
328 dma_async_client_register(&async_tx_dma
);
329 dma_async_client_chan_request(&async_tx_dma
);
331 printk(KERN_INFO
"async_tx: api initialized (async)\n");
335 printk(KERN_ERR
"async_tx: initialization failure\n");
338 free_percpu(channel_table
[cap
]);
343 static void __exit
async_tx_exit(void)
345 enum dma_transaction_type cap
;
347 channel_table_initialized
= 0;
349 for_each_dma_cap_mask(cap
, dma_cap_mask_all
)
350 if (channel_table
[cap
])
351 free_percpu(channel_table
[cap
]);
353 dma_async_client_unregister(&async_tx_dma
);
357 * async_tx_find_channel - find a channel to carry out the operation or let
358 * the transaction execute synchronously
359 * @depend_tx: transaction dependency
360 * @tx_type: transaction type
363 async_tx_find_channel(struct dma_async_tx_descriptor
*depend_tx
,
364 enum dma_transaction_type tx_type
)
366 /* see if we can keep the chain on one channel */
368 dma_has_cap(tx_type
, depend_tx
->chan
->device
->cap_mask
))
369 return depend_tx
->chan
;
370 else if (likely(channel_table_initialized
)) {
371 struct dma_chan_ref
*ref
;
373 ref
= per_cpu_ptr(channel_table
[tx_type
], cpu
)->ref
;
375 return ref
? ref
->chan
: NULL
;
379 EXPORT_SYMBOL_GPL(async_tx_find_channel
);
381 static int __init
async_tx_init(void)
383 printk(KERN_INFO
"async_tx: api initialized (sync-only)\n");
387 static void __exit
async_tx_exit(void)
394 async_tx_submit(struct dma_chan
*chan
, struct dma_async_tx_descriptor
*tx
,
395 enum async_tx_flags flags
, struct dma_async_tx_descriptor
*depend_tx
,
396 dma_async_tx_callback cb_fn
, void *cb_param
)
398 tx
->callback
= cb_fn
;
399 tx
->callback_param
= cb_param
;
401 /* set this new tx to run after depend_tx if:
402 * 1/ a dependency exists (depend_tx is !NULL)
403 * 2/ the tx can not be submitted to the current channel
405 if (depend_tx
&& depend_tx
->chan
!= chan
) {
406 /* if ack is already set then we cannot be sure
407 * we are referring to the correct operation
409 BUG_ON(depend_tx
->ack
);
411 tx
->parent
= depend_tx
;
412 spin_lock_bh(&depend_tx
->lock
);
413 list_add_tail(&tx
->depend_node
, &depend_tx
->depend_list
);
414 if (depend_tx
->cookie
== 0) {
415 struct dma_chan
*dep_chan
= depend_tx
->chan
;
416 struct dma_device
*dep_dev
= dep_chan
->device
;
417 dep_dev
->device_dependency_added(dep_chan
);
419 spin_unlock_bh(&depend_tx
->lock
);
421 /* schedule an interrupt to trigger the channel switch */
422 async_trigger_callback(ASYNC_TX_ACK
, depend_tx
, NULL
, NULL
);
428 if (flags
& ASYNC_TX_ACK
)
431 if (depend_tx
&& (flags
& ASYNC_TX_DEP_ACK
))
432 async_tx_ack(depend_tx
);
434 EXPORT_SYMBOL_GPL(async_tx_submit
);
437 * async_trigger_callback - schedules the callback function to be run after
438 * any dependent operations have been completed.
439 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
440 * @depend_tx: 'callback' requires the completion of this transaction
441 * @cb_fn: function to call after depend_tx completes
442 * @cb_param: parameter to pass to the callback routine
444 struct dma_async_tx_descriptor
*
445 async_trigger_callback(enum async_tx_flags flags
,
446 struct dma_async_tx_descriptor
*depend_tx
,
447 dma_async_tx_callback cb_fn
, void *cb_param
)
449 struct dma_chan
*chan
;
450 struct dma_device
*device
;
451 struct dma_async_tx_descriptor
*tx
;
454 chan
= depend_tx
->chan
;
455 device
= chan
->device
;
457 /* see if we can schedule an interrupt
458 * otherwise poll for completion
460 if (device
&& !dma_has_cap(DMA_INTERRUPT
, device
->cap_mask
))
463 tx
= device
? device
->device_prep_dma_interrupt(chan
) : NULL
;
468 pr_debug("%s: (async)\n", __FUNCTION__
);
470 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
472 pr_debug("%s: (sync)\n", __FUNCTION__
);
474 /* wait for any prerequisite operations */
476 /* if ack is already set then we cannot be sure
477 * we are referring to the correct operation
479 BUG_ON(depend_tx
->ack
);
480 if (dma_wait_for_async_tx(depend_tx
) == DMA_ERROR
)
481 panic("%s: DMA_ERROR waiting for depend_tx\n",
485 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
490 EXPORT_SYMBOL_GPL(async_trigger_callback
);
492 module_init(async_tx_init
);
493 module_exit(async_tx_exit
);
495 MODULE_AUTHOR("Intel Corporation");
496 MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
497 MODULE_LICENSE("GPL");