crypto/async_tx/async_tx.c

   1 /*
   2  * core routines for the asynchronous memory transfer/transform api
   3  *
   4  * Copyright © 2006, Intel Corporation.
   5  *
   6  *      Dan Williams <dan.j.williams@intel.com>
   7  *
   8  *      with architecture considerations by:
   9  *      Neil Brown <neilb@suse.de>
  10  *      Jeff Garzik <jeff@garzik.org>
  11  *
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms and conditions of the GNU General Public License,
  14  * version 2, as published by the Free Software Foundation.
  15  *
  16  * This program is distributed in the hope it will be useful, but WITHOUT
  17  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  18  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  19  * more details.
  20  *
  21  * You should have received a copy of the GNU General Public License along with
  22  * this program; if not, write to the Free Software Foundation, Inc.,
  23  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  24  *
  25  */
  26 #include <linux/rculist.h>
  27 #include <linux/kernel.h>
  28 #include <linux/async_tx.h>
  29
  30 #ifdef CONFIG_DMA_ENGINE
  31 static enum dma_state_client
  32 dma_channel_add_remove(struct dma_client *client,
  33         struct dma_chan *chan, enum dma_state state);
  34
  35 static struct dma_client async_tx_dma = {
  36         .event_callback = dma_channel_add_remove,
  37         /* .cap_mask == 0 defaults to all channels */
  38 };
  39
  40 /**
  41  * async_tx_lock - protect modification of async_tx_master_list and serialize
  42  *      rebalance operations
  43  */
  44 static DEFINE_SPINLOCK(async_tx_lock);
  45
  46 static LIST_HEAD(async_tx_master_list);
  47
  48 static void
  49 free_dma_chan_ref(struct rcu_head *rcu)
  50 {
  51         struct dma_chan_ref *ref;
  52         ref = container_of(rcu, struct dma_chan_ref, rcu);
  53         kfree(ref);
  54 }
  55
  56 static void
  57 init_dma_chan_ref(struct dma_chan_ref *ref, struct dma_chan *chan)
  58 {
  59         INIT_LIST_HEAD(&ref->node);
  60         INIT_RCU_HEAD(&ref->rcu);
  61         ref->chan = chan;
  62         atomic_set(&ref->count, 0);
  63 }
  64
  65 static enum dma_state_client
  66 dma_channel_add_remove(struct dma_client *client,
  67         struct dma_chan *chan, enum dma_state state)
  68 {
  69         unsigned long found, flags;
  70         struct dma_chan_ref *master_ref, *ref;
  71         enum dma_state_client ack = DMA_DUP; /* default: take no action */
  72
  73         switch (state) {
  74         case DMA_RESOURCE_AVAILABLE:
  75                 found = 0;
  76                 rcu_read_lock();
  77                 list_for_each_entry_rcu(ref, &async_tx_master_list, node)
  78                         if (ref->chan == chan) {
  79                                 found = 1;
  80                                 break;
  81                         }
  82                 rcu_read_unlock();
  83
  84                 pr_debug("async_tx: dma resource available [%s]\n",
  85                         found ? "old" : "new");
  86
  87                 if (!found)
  88                         ack = DMA_ACK;
  89                 else
  90                         break;
  91
  92                 /* add the channel to the generic management list */
  93                 master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL);
  94                 if (master_ref) {
  95                         init_dma_chan_ref(master_ref, chan);
  96                         spin_lock_irqsave(&async_tx_lock, flags);
  97                         list_add_tail_rcu(&master_ref->node,
  98                                 &async_tx_master_list);
  99                         spin_unlock_irqrestore(&async_tx_lock,
 100                                 flags);
 101                 } else {
 102                         printk(KERN_WARNING "async_tx: unable to create"
 103                                 " new master entry in response to"
 104                                 " a DMA_RESOURCE_ADDED event"
 105                                 " (-ENOMEM)\n");
 106                         return 0;
 107                 }
 108                 break;
 109         case DMA_RESOURCE_REMOVED:
 110                 found = 0;
 111                 spin_lock_irqsave(&async_tx_lock, flags);
 112                 list_for_each_entry(ref, &async_tx_master_list, node)
 113                         if (ref->chan == chan) {
 114                                 list_del_rcu(&ref->node);
 115                                 call_rcu(&ref->rcu, free_dma_chan_ref);
 116                                 found = 1;
 117                                 break;
 118                         }
 119                 spin_unlock_irqrestore(&async_tx_lock, flags);
 120
 121                 pr_debug("async_tx: dma resource removed [%s]\n",
 122                         found ? "ours" : "not ours");
 123
 124                 if (found)
 125                         ack = DMA_ACK;
 126                 else
 127                         break;
 128                 break;
 129         case DMA_RESOURCE_SUSPEND:
 130         case DMA_RESOURCE_RESUME:
 131                 printk(KERN_WARNING "async_tx: does not support dma channel"
 132                         " suspend/resume\n");
 133                 break;
 134         default:
 135                 BUG();
 136         }
 137
 138         return ack;
 139 }
 140
 141 static int __init async_tx_init(void)
 142 {
 143         dma_async_client_register(&async_tx_dma);
 144         dma_async_client_chan_request(&async_tx_dma);
 145
 146         printk(KERN_INFO "async_tx: api initialized (async)\n");
 147
 148         return 0;
 149 }
 150
 151 static void __exit async_tx_exit(void)
 152 {
 153         dma_async_client_unregister(&async_tx_dma);
 154 }
 155
 156 /**
 157  * __async_tx_find_channel - find a channel to carry out the operation or let
 158  *      the transaction execute synchronously
 159  * @depend_tx: transaction dependency
 160  * @tx_type: transaction type
 161  */
 162 struct dma_chan *
 163 __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
 164         enum dma_transaction_type tx_type)
 165 {
 166         /* see if we can keep the chain on one channel */
 167         if (depend_tx &&
 168             dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
 169                 return depend_tx->chan;
 170         return dma_find_channel(tx_type);
 171 }
 172 EXPORT_SYMBOL_GPL(__async_tx_find_channel);
 173 #else
 174 static int __init async_tx_init(void)
 175 {
 176         printk(KERN_INFO "async_tx: api initialized (sync-only)\n");
 177         return 0;
 178 }
 179
 180 static void __exit async_tx_exit(void)
 181 {
 182         do { } while (0);
 183 }
 184 #endif
 185
 186
 187 /**
 188  * async_tx_channel_switch - queue an interrupt descriptor with a dependency
 189  *      pre-attached.
 190  * @depend_tx: the operation that must finish before the new operation runs
 191  * @tx: the new operation
 192  */
 193 static void
 194 async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
 195                         struct dma_async_tx_descriptor *tx)
 196 {
 197         struct dma_chan *chan;
 198         struct dma_device *device;
 199         struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
 200
 201         /* first check to see if we can still append to depend_tx */
 202         spin_lock_bh(&depend_tx->lock);
 203         if (depend_tx->parent && depend_tx->chan == tx->chan) {
 204                 tx->parent = depend_tx;
 205                 depend_tx->next = tx;
 206                 intr_tx = NULL;
 207         }
 208         spin_unlock_bh(&depend_tx->lock);
 209
 210         if (!intr_tx)
 211                 return;
 212
 213         chan = depend_tx->chan;
 214         device = chan->device;
 215
 216         /* see if we can schedule an interrupt
 217          * otherwise poll for completion
 218          */
 219         if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
 220                 intr_tx = device->device_prep_dma_interrupt(chan, 0);
 221         else
 222                 intr_tx = NULL;
 223
 224         if (intr_tx) {
 225                 intr_tx->callback = NULL;
 226                 intr_tx->callback_param = NULL;
 227                 tx->parent = intr_tx;
 228                 /* safe to set ->next outside the lock since we know we are
 229                  * not submitted yet
 230                  */
 231                 intr_tx->next = tx;
 232
 233                 /* check if we need to append */
 234                 spin_lock_bh(&depend_tx->lock);
 235                 if (depend_tx->parent) {
 236                         intr_tx->parent = depend_tx;
 237                         depend_tx->next = intr_tx;
 238                         async_tx_ack(intr_tx);
 239                         intr_tx = NULL;
 240                 }
 241                 spin_unlock_bh(&depend_tx->lock);
 242
 243                 if (intr_tx) {
 244                         intr_tx->parent = NULL;
 245                         intr_tx->tx_submit(intr_tx);
 246                         async_tx_ack(intr_tx);
 247                 }
 248         } else {
 249                 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
 250                         panic("%s: DMA_ERROR waiting for depend_tx\n",
 251                               __func__);
 252                 tx->tx_submit(tx);
 253         }
 254 }
 255
 256
 257 /**
 258  * submit_disposition - while holding depend_tx->lock we must avoid submitting
 259  *      new operations to prevent a circular locking dependency with
 260  *      drivers that already hold a channel lock when calling
 261  *      async_tx_run_dependencies.
 262  * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
 263  * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
 264  * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
 265  */
 266 enum submit_disposition {
 267         ASYNC_TX_SUBMITTED,
 268         ASYNC_TX_CHANNEL_SWITCH,
 269         ASYNC_TX_DIRECT_SUBMIT,
 270 };
 271
 272 void
 273 async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
 274         enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
 275         dma_async_tx_callback cb_fn, void *cb_param)
 276 {
 277         tx->callback = cb_fn;
 278         tx->callback_param = cb_param;
 279
 280         if (depend_tx) {
 281                 enum submit_disposition s;
 282
 283                 /* sanity check the dependency chain:
 284                  * 1/ if ack is already set then we cannot be sure
 285                  * we are referring to the correct operation
 286                  * 2/ dependencies are 1:1 i.e. two transactions can
 287                  * not depend on the same parent
 288                  */
 289                 BUG_ON(async_tx_test_ack(depend_tx) || depend_tx->next ||
 290                        tx->parent);
 291
 292                 /* the lock prevents async_tx_run_dependencies from missing
 293                  * the setting of ->next when ->parent != NULL
 294                  */
 295                 spin_lock_bh(&depend_tx->lock);
 296                 if (depend_tx->parent) {
 297                         /* we have a parent so we can not submit directly
 298                          * if we are staying on the same channel: append
 299                          * else: channel switch
 300                          */
 301                         if (depend_tx->chan == chan) {
 302                                 tx->parent = depend_tx;
 303                                 depend_tx->next = tx;
 304                                 s = ASYNC_TX_SUBMITTED;
 305                         } else
 306                                 s = ASYNC_TX_CHANNEL_SWITCH;
 307                 } else {
 308                         /* we do not have a parent so we may be able to submit
 309                          * directly if we are staying on the same channel
 310                          */
 311                         if (depend_tx->chan == chan)
 312                                 s = ASYNC_TX_DIRECT_SUBMIT;
 313                         else
 314                                 s = ASYNC_TX_CHANNEL_SWITCH;
 315                 }
 316                 spin_unlock_bh(&depend_tx->lock);
 317
 318                 switch (s) {
 319                 case ASYNC_TX_SUBMITTED:
 320                         break;
 321                 case ASYNC_TX_CHANNEL_SWITCH:
 322                         async_tx_channel_switch(depend_tx, tx);
 323                         break;
 324                 case ASYNC_TX_DIRECT_SUBMIT:
 325                         tx->parent = NULL;
 326                         tx->tx_submit(tx);
 327                         break;
 328                 }
 329         } else {
 330                 tx->parent = NULL;
 331                 tx->tx_submit(tx);
 332         }
 333
 334         if (flags & ASYNC_TX_ACK)
 335                 async_tx_ack(tx);
 336
 337         if (depend_tx && (flags & ASYNC_TX_DEP_ACK))
 338                 async_tx_ack(depend_tx);
 339 }
 340 EXPORT_SYMBOL_GPL(async_tx_submit);
 341
 342 /**
 343  * async_trigger_callback - schedules the callback function to be run after
 344  * any dependent operations have been completed.
 345  * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
 346  * @depend_tx: 'callback' requires the completion of this transaction
 347  * @cb_fn: function to call after depend_tx completes
 348  * @cb_param: parameter to pass to the callback routine
 349  */
 350 struct dma_async_tx_descriptor *
 351 async_trigger_callback(enum async_tx_flags flags,
 352         struct dma_async_tx_descriptor *depend_tx,
 353         dma_async_tx_callback cb_fn, void *cb_param)
 354 {
 355         struct dma_chan *chan;
 356         struct dma_device *device;
 357         struct dma_async_tx_descriptor *tx;
 358
 359         if (depend_tx) {
 360                 chan = depend_tx->chan;
 361                 device = chan->device;
 362
 363                 /* see if we can schedule an interrupt
 364                  * otherwise poll for completion
 365                  */
 366                 if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
 367                         device = NULL;
 368
 369                 tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
 370         } else
 371                 tx = NULL;
 372
 373         if (tx) {
 374                 pr_debug("%s: (async)\n", __func__);
 375
 376                 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
 377         } else {
 378                 pr_debug("%s: (sync)\n", __func__);
 379
 380                 /* wait for any prerequisite operations */
 381                 async_tx_quiesce(&depend_tx);
 382
 383                 async_tx_sync_epilog(cb_fn, cb_param);
 384         }
 385
 386         return tx;
 387 }
 388 EXPORT_SYMBOL_GPL(async_trigger_callback);
 389
 390 /**
 391  * async_tx_quiesce - ensure tx is complete and freeable upon return
 392  * @tx - transaction to quiesce
 393  */
 394 void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
 395 {
 396         if (*tx) {
 397                 /* if ack is already set then we cannot be sure
 398                  * we are referring to the correct operation
 399                  */
 400                 BUG_ON(async_tx_test_ack(*tx));
 401                 if (dma_wait_for_async_tx(*tx) == DMA_ERROR)
 402                         panic("DMA_ERROR waiting for transaction\n");
 403                 async_tx_ack(*tx);
 404                 *tx = NULL;
 405         }
 406 }
 407 EXPORT_SYMBOL_GPL(async_tx_quiesce);
 408
 409 module_init(async_tx_init);
 410 module_exit(async_tx_exit);
 411
 412 MODULE_AUTHOR("Intel Corporation");
 413 MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
 414 MODULE_LICENSE("GPL");