4 * Support for OMAP AES HW acceleration.
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8 * Copyright (c) 2011 Texas Instruments Incorporated
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
16 #define pr_fmt(fmt) "%20s: " fmt, __func__
17 #define prn(num) pr_debug(#num "=%d\n", num)
18 #define prx(num) pr_debug(#num "=%x\n", num)
20 #include <linux/err.h>
21 #include <linux/module.h>
22 #include <linux/init.h>
23 #include <linux/errno.h>
24 #include <linux/kernel.h>
25 #include <linux/platform_device.h>
26 #include <linux/scatterlist.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/dmaengine.h>
29 #include <linux/pm_runtime.h>
31 #include <linux/of_device.h>
32 #include <linux/of_address.h>
34 #include <linux/crypto.h>
35 #include <linux/interrupt.h>
36 #include <crypto/scatterwalk.h>
37 #include <crypto/aes.h>
38 #include <crypto/algapi.h>
39 #include <crypto/engine.h>
41 #define DST_MAXBURST 4
42 #define DMA_MIN (DST_MAXBURST * sizeof(u32))
44 #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
46 /* OMAP TRM gives bitfields as start:end, where start is the higher bit
47 number. For example 7:0 */
48 #define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end))
49 #define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
51 #define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \
53 #define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04))
55 #define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs)
56 #define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7)
57 #define AES_REG_CTRL_CTR_WIDTH_32 0
58 #define AES_REG_CTRL_CTR_WIDTH_64 BIT(7)
59 #define AES_REG_CTRL_CTR_WIDTH_96 BIT(8)
60 #define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7)
61 #define AES_REG_CTRL_CTR BIT(6)
62 #define AES_REG_CTRL_CBC BIT(5)
63 #define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3)
64 #define AES_REG_CTRL_DIRECTION BIT(2)
65 #define AES_REG_CTRL_INPUT_READY BIT(1)
66 #define AES_REG_CTRL_OUTPUT_READY BIT(0)
67 #define AES_REG_CTRL_MASK GENMASK(24, 2)
69 #define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04))
71 #define AES_REG_REV(dd) ((dd)->pdata->rev_ofs)
73 #define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs)
74 #define AES_REG_MASK_SIDLE BIT(6)
75 #define AES_REG_MASK_START BIT(5)
76 #define AES_REG_MASK_DMA_OUT_EN BIT(3)
77 #define AES_REG_MASK_DMA_IN_EN BIT(2)
78 #define AES_REG_MASK_SOFTRESET BIT(1)
79 #define AES_REG_AUTOIDLE BIT(0)
81 #define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04))
83 #define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs)
84 #define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs)
85 #define AES_REG_IRQ_DATA_IN BIT(1)
86 #define AES_REG_IRQ_DATA_OUT BIT(2)
87 #define DEFAULT_TIMEOUT (5*HZ)
89 #define FLAGS_MODE_MASK 0x000f
90 #define FLAGS_ENCRYPT BIT(0)
91 #define FLAGS_CBC BIT(1)
92 #define FLAGS_GIV BIT(2)
93 #define FLAGS_CTR BIT(3)
95 #define FLAGS_INIT BIT(4)
96 #define FLAGS_FAST BIT(5)
97 #define FLAGS_BUSY BIT(6)
99 #define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2)
101 struct omap_aes_ctx
{
102 struct omap_aes_dev
*dd
;
105 u32 key
[AES_KEYSIZE_256
/ sizeof(u32
)];
109 struct omap_aes_reqctx
{
113 #define OMAP_AES_QUEUE_LENGTH 1
114 #define OMAP_AES_CACHE_SIZE 0
116 struct omap_aes_algs_info
{
117 struct crypto_alg
*algs_list
;
119 unsigned int registered
;
122 struct omap_aes_pdata
{
123 struct omap_aes_algs_info
*algs_info
;
124 unsigned int algs_info_size
;
126 void (*trigger
)(struct omap_aes_dev
*dd
, int length
);
147 struct omap_aes_dev
{
148 struct list_head list
;
149 unsigned long phys_base
;
150 void __iomem
*io_base
;
151 struct omap_aes_ctx
*ctx
;
156 struct tasklet_struct done_task
;
158 struct ablkcipher_request
*req
;
159 struct crypto_engine
*engine
;
162 * total is used by PIO mode for book keeping so introduce
163 * variable total_save as need it to calc page_order
168 struct scatterlist
*in_sg
;
169 struct scatterlist
*out_sg
;
171 /* Buffers for copying for unaligned cases */
172 struct scatterlist in_sgl
;
173 struct scatterlist out_sgl
;
174 struct scatterlist
*orig_out
;
177 struct scatter_walk in_walk
;
178 struct scatter_walk out_walk
;
179 struct dma_chan
*dma_lch_in
;
180 struct dma_chan
*dma_lch_out
;
184 const struct omap_aes_pdata
*pdata
;
187 /* keep registered devices data here */
188 static LIST_HEAD(dev_list
);
189 static DEFINE_SPINLOCK(list_lock
);
192 #define omap_aes_read(dd, offset) \
195 _read_ret = __raw_readl(dd->io_base + offset); \
196 pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \
197 offset, _read_ret); \
201 static inline u32
omap_aes_read(struct omap_aes_dev
*dd
, u32 offset
)
203 return __raw_readl(dd
->io_base
+ offset
);
208 #define omap_aes_write(dd, offset, value) \
210 pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
212 __raw_writel(value, dd->io_base + offset); \
215 static inline void omap_aes_write(struct omap_aes_dev
*dd
, u32 offset
,
218 __raw_writel(value
, dd
->io_base
+ offset
);
222 static inline void omap_aes_write_mask(struct omap_aes_dev
*dd
, u32 offset
,
227 val
= omap_aes_read(dd
, offset
);
230 omap_aes_write(dd
, offset
, val
);
233 static void omap_aes_write_n(struct omap_aes_dev
*dd
, u32 offset
,
234 u32
*value
, int count
)
236 for (; count
--; value
++, offset
+= 4)
237 omap_aes_write(dd
, offset
, *value
);
240 static int omap_aes_hw_init(struct omap_aes_dev
*dd
)
242 if (!(dd
->flags
& FLAGS_INIT
)) {
243 dd
->flags
|= FLAGS_INIT
;
250 static int omap_aes_write_ctrl(struct omap_aes_dev
*dd
)
256 err
= omap_aes_hw_init(dd
);
260 key32
= dd
->ctx
->keylen
/ sizeof(u32
);
262 /* it seems a key should always be set even if it has not changed */
263 for (i
= 0; i
< key32
; i
++) {
264 omap_aes_write(dd
, AES_REG_KEY(dd
, i
),
265 __le32_to_cpu(dd
->ctx
->key
[i
]));
268 if ((dd
->flags
& (FLAGS_CBC
| FLAGS_CTR
)) && dd
->req
->info
)
269 omap_aes_write_n(dd
, AES_REG_IV(dd
, 0), dd
->req
->info
, 4);
271 val
= FLD_VAL(((dd
->ctx
->keylen
>> 3) - 1), 4, 3);
272 if (dd
->flags
& FLAGS_CBC
)
273 val
|= AES_REG_CTRL_CBC
;
274 if (dd
->flags
& FLAGS_CTR
)
275 val
|= AES_REG_CTRL_CTR
| AES_REG_CTRL_CTR_WIDTH_128
;
277 if (dd
->flags
& FLAGS_ENCRYPT
)
278 val
|= AES_REG_CTRL_DIRECTION
;
280 omap_aes_write_mask(dd
, AES_REG_CTRL(dd
), val
, AES_REG_CTRL_MASK
);
285 static void omap_aes_dma_trigger_omap2(struct omap_aes_dev
*dd
, int length
)
289 val
= dd
->pdata
->dma_start
;
291 if (dd
->dma_lch_out
!= NULL
)
292 val
|= dd
->pdata
->dma_enable_out
;
293 if (dd
->dma_lch_in
!= NULL
)
294 val
|= dd
->pdata
->dma_enable_in
;
296 mask
= dd
->pdata
->dma_enable_out
| dd
->pdata
->dma_enable_in
|
297 dd
->pdata
->dma_start
;
299 omap_aes_write_mask(dd
, AES_REG_MASK(dd
), val
, mask
);
303 static void omap_aes_dma_trigger_omap4(struct omap_aes_dev
*dd
, int length
)
305 omap_aes_write(dd
, AES_REG_LENGTH_N(0), length
);
306 omap_aes_write(dd
, AES_REG_LENGTH_N(1), 0);
308 omap_aes_dma_trigger_omap2(dd
, length
);
311 static void omap_aes_dma_stop(struct omap_aes_dev
*dd
)
315 mask
= dd
->pdata
->dma_enable_out
| dd
->pdata
->dma_enable_in
|
316 dd
->pdata
->dma_start
;
318 omap_aes_write_mask(dd
, AES_REG_MASK(dd
), 0, mask
);
321 static struct omap_aes_dev
*omap_aes_find_dev(struct omap_aes_ctx
*ctx
)
323 struct omap_aes_dev
*dd
= NULL
, *tmp
;
325 spin_lock_bh(&list_lock
);
327 list_for_each_entry(tmp
, &dev_list
, list
) {
328 /* FIXME: take fist available aes core */
334 /* already found before */
337 spin_unlock_bh(&list_lock
);
342 static void omap_aes_dma_out_callback(void *data
)
344 struct omap_aes_dev
*dd
= data
;
346 /* dma_lch_out - completed */
347 tasklet_schedule(&dd
->done_task
);
350 static int omap_aes_dma_init(struct omap_aes_dev
*dd
)
354 dd
->dma_lch_out
= NULL
;
355 dd
->dma_lch_in
= NULL
;
357 dd
->dma_lch_in
= dma_request_chan(dd
->dev
, "rx");
358 if (IS_ERR(dd
->dma_lch_in
)) {
359 dev_err(dd
->dev
, "Unable to request in DMA channel\n");
360 return PTR_ERR(dd
->dma_lch_in
);
363 dd
->dma_lch_out
= dma_request_chan(dd
->dev
, "tx");
364 if (IS_ERR(dd
->dma_lch_out
)) {
365 dev_err(dd
->dev
, "Unable to request out DMA channel\n");
366 err
= PTR_ERR(dd
->dma_lch_out
);
373 dma_release_channel(dd
->dma_lch_in
);
378 static void omap_aes_dma_cleanup(struct omap_aes_dev
*dd
)
383 dma_release_channel(dd
->dma_lch_out
);
384 dma_release_channel(dd
->dma_lch_in
);
387 static void sg_copy_buf(void *buf
, struct scatterlist
*sg
,
388 unsigned int start
, unsigned int nbytes
, int out
)
390 struct scatter_walk walk
;
395 scatterwalk_start(&walk
, sg
);
396 scatterwalk_advance(&walk
, start
);
397 scatterwalk_copychunks(buf
, &walk
, nbytes
, out
);
398 scatterwalk_done(&walk
, out
, 0);
401 static int omap_aes_crypt_dma(struct crypto_tfm
*tfm
,
402 struct scatterlist
*in_sg
, struct scatterlist
*out_sg
,
403 int in_sg_len
, int out_sg_len
)
405 struct omap_aes_ctx
*ctx
= crypto_tfm_ctx(tfm
);
406 struct omap_aes_dev
*dd
= ctx
->dd
;
407 struct dma_async_tx_descriptor
*tx_in
, *tx_out
;
408 struct dma_slave_config cfg
;
412 scatterwalk_start(&dd
->in_walk
, dd
->in_sg
);
413 scatterwalk_start(&dd
->out_walk
, dd
->out_sg
);
415 /* Enable DATAIN interrupt and let it take
417 omap_aes_write(dd
, AES_REG_IRQ_ENABLE(dd
), 0x2);
421 dma_sync_sg_for_device(dd
->dev
, dd
->in_sg
, in_sg_len
, DMA_TO_DEVICE
);
423 memset(&cfg
, 0, sizeof(cfg
));
425 cfg
.src_addr
= dd
->phys_base
+ AES_REG_DATA_N(dd
, 0);
426 cfg
.dst_addr
= dd
->phys_base
+ AES_REG_DATA_N(dd
, 0);
427 cfg
.src_addr_width
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
428 cfg
.dst_addr_width
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
429 cfg
.src_maxburst
= DST_MAXBURST
;
430 cfg
.dst_maxburst
= DST_MAXBURST
;
433 ret
= dmaengine_slave_config(dd
->dma_lch_in
, &cfg
);
435 dev_err(dd
->dev
, "can't configure IN dmaengine slave: %d\n",
440 tx_in
= dmaengine_prep_slave_sg(dd
->dma_lch_in
, in_sg
, in_sg_len
,
442 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
444 dev_err(dd
->dev
, "IN prep_slave_sg() failed\n");
448 /* No callback necessary */
449 tx_in
->callback_param
= dd
;
452 ret
= dmaengine_slave_config(dd
->dma_lch_out
, &cfg
);
454 dev_err(dd
->dev
, "can't configure OUT dmaengine slave: %d\n",
459 tx_out
= dmaengine_prep_slave_sg(dd
->dma_lch_out
, out_sg
, out_sg_len
,
461 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
463 dev_err(dd
->dev
, "OUT prep_slave_sg() failed\n");
467 tx_out
->callback
= omap_aes_dma_out_callback
;
468 tx_out
->callback_param
= dd
;
470 dmaengine_submit(tx_in
);
471 dmaengine_submit(tx_out
);
473 dma_async_issue_pending(dd
->dma_lch_in
);
474 dma_async_issue_pending(dd
->dma_lch_out
);
477 dd
->pdata
->trigger(dd
, dd
->total
);
482 static int omap_aes_crypt_dma_start(struct omap_aes_dev
*dd
)
484 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(
485 crypto_ablkcipher_reqtfm(dd
->req
));
488 pr_debug("total: %d\n", dd
->total
);
491 err
= dma_map_sg(dd
->dev
, dd
->in_sg
, dd
->in_sg_len
,
494 dev_err(dd
->dev
, "dma_map_sg() error\n");
498 err
= dma_map_sg(dd
->dev
, dd
->out_sg
, dd
->out_sg_len
,
501 dev_err(dd
->dev
, "dma_map_sg() error\n");
506 err
= omap_aes_crypt_dma(tfm
, dd
->in_sg
, dd
->out_sg
, dd
->in_sg_len
,
508 if (err
&& !dd
->pio_only
) {
509 dma_unmap_sg(dd
->dev
, dd
->in_sg
, dd
->in_sg_len
, DMA_TO_DEVICE
);
510 dma_unmap_sg(dd
->dev
, dd
->out_sg
, dd
->out_sg_len
,
517 static void omap_aes_finish_req(struct omap_aes_dev
*dd
, int err
)
519 struct ablkcipher_request
*req
= dd
->req
;
521 pr_debug("err: %d\n", err
);
523 crypto_finalize_cipher_request(dd
->engine
, req
, err
);
526 static int omap_aes_crypt_dma_stop(struct omap_aes_dev
*dd
)
528 pr_debug("total: %d\n", dd
->total
);
530 omap_aes_dma_stop(dd
);
536 static int omap_aes_check_aligned(struct scatterlist
*sg
, int total
)
540 if (!IS_ALIGNED(total
, AES_BLOCK_SIZE
))
544 if (!IS_ALIGNED(sg
->offset
, 4))
546 if (!IS_ALIGNED(sg
->length
, AES_BLOCK_SIZE
))
559 static int omap_aes_copy_sgs(struct omap_aes_dev
*dd
)
561 void *buf_in
, *buf_out
;
564 total
= ALIGN(dd
->total
, AES_BLOCK_SIZE
);
565 pages
= get_order(total
);
567 buf_in
= (void *)__get_free_pages(GFP_ATOMIC
, pages
);
568 buf_out
= (void *)__get_free_pages(GFP_ATOMIC
, pages
);
570 if (!buf_in
|| !buf_out
) {
571 pr_err("Couldn't allocated pages for unaligned cases.\n");
575 dd
->orig_out
= dd
->out_sg
;
577 sg_copy_buf(buf_in
, dd
->in_sg
, 0, dd
->total
, 0);
579 sg_init_table(&dd
->in_sgl
, 1);
580 sg_set_buf(&dd
->in_sgl
, buf_in
, total
);
581 dd
->in_sg
= &dd
->in_sgl
;
584 sg_init_table(&dd
->out_sgl
, 1);
585 sg_set_buf(&dd
->out_sgl
, buf_out
, total
);
586 dd
->out_sg
= &dd
->out_sgl
;
592 static int omap_aes_handle_queue(struct omap_aes_dev
*dd
,
593 struct ablkcipher_request
*req
)
596 return crypto_transfer_cipher_request_to_engine(dd
->engine
, req
);
601 static int omap_aes_prepare_req(struct crypto_engine
*engine
,
602 struct ablkcipher_request
*req
)
604 struct omap_aes_ctx
*ctx
= crypto_ablkcipher_ctx(
605 crypto_ablkcipher_reqtfm(req
));
606 struct omap_aes_dev
*dd
= omap_aes_find_dev(ctx
);
607 struct omap_aes_reqctx
*rctx
;
612 /* assign new request to device */
614 dd
->total
= req
->nbytes
;
615 dd
->total_save
= req
->nbytes
;
616 dd
->in_sg
= req
->src
;
617 dd
->out_sg
= req
->dst
;
619 dd
->in_sg_len
= sg_nents_for_len(dd
->in_sg
, dd
->total
);
620 if (dd
->in_sg_len
< 0)
621 return dd
->in_sg_len
;
623 dd
->out_sg_len
= sg_nents_for_len(dd
->out_sg
, dd
->total
);
624 if (dd
->out_sg_len
< 0)
625 return dd
->out_sg_len
;
627 if (omap_aes_check_aligned(dd
->in_sg
, dd
->total
) ||
628 omap_aes_check_aligned(dd
->out_sg
, dd
->total
)) {
629 if (omap_aes_copy_sgs(dd
))
630 pr_err("Failed to copy SGs for unaligned cases\n");
636 rctx
= ablkcipher_request_ctx(req
);
637 ctx
= crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req
));
638 rctx
->mode
&= FLAGS_MODE_MASK
;
639 dd
->flags
= (dd
->flags
& ~FLAGS_MODE_MASK
) | rctx
->mode
;
644 return omap_aes_write_ctrl(dd
);
647 static int omap_aes_crypt_req(struct crypto_engine
*engine
,
648 struct ablkcipher_request
*req
)
650 struct omap_aes_ctx
*ctx
= crypto_ablkcipher_ctx(
651 crypto_ablkcipher_reqtfm(req
));
652 struct omap_aes_dev
*dd
= omap_aes_find_dev(ctx
);
657 return omap_aes_crypt_dma_start(dd
);
660 static void omap_aes_done_task(unsigned long data
)
662 struct omap_aes_dev
*dd
= (struct omap_aes_dev
*)data
;
663 void *buf_in
, *buf_out
;
666 pr_debug("enter done_task\n");
669 dma_sync_sg_for_device(dd
->dev
, dd
->out_sg
, dd
->out_sg_len
,
671 dma_unmap_sg(dd
->dev
, dd
->in_sg
, dd
->in_sg_len
, DMA_TO_DEVICE
);
672 dma_unmap_sg(dd
->dev
, dd
->out_sg
, dd
->out_sg_len
,
674 omap_aes_crypt_dma_stop(dd
);
677 if (dd
->sgs_copied
) {
678 buf_in
= sg_virt(&dd
->in_sgl
);
679 buf_out
= sg_virt(&dd
->out_sgl
);
681 sg_copy_buf(buf_out
, dd
->orig_out
, 0, dd
->total_save
, 1);
683 len
= ALIGN(dd
->total_save
, AES_BLOCK_SIZE
);
684 pages
= get_order(len
);
685 free_pages((unsigned long)buf_in
, pages
);
686 free_pages((unsigned long)buf_out
, pages
);
689 omap_aes_finish_req(dd
, 0);
694 static int omap_aes_crypt(struct ablkcipher_request
*req
, unsigned long mode
)
696 struct omap_aes_ctx
*ctx
= crypto_ablkcipher_ctx(
697 crypto_ablkcipher_reqtfm(req
));
698 struct omap_aes_reqctx
*rctx
= ablkcipher_request_ctx(req
);
699 struct omap_aes_dev
*dd
;
701 pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req
->nbytes
,
702 !!(mode
& FLAGS_ENCRYPT
),
703 !!(mode
& FLAGS_CBC
));
705 dd
= omap_aes_find_dev(ctx
);
711 return omap_aes_handle_queue(dd
, req
);
714 /* ********************** ALG API ************************************ */
716 static int omap_aes_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
719 struct omap_aes_ctx
*ctx
= crypto_ablkcipher_ctx(tfm
);
721 if (keylen
!= AES_KEYSIZE_128
&& keylen
!= AES_KEYSIZE_192
&&
722 keylen
!= AES_KEYSIZE_256
)
725 pr_debug("enter, keylen: %d\n", keylen
);
727 memcpy(ctx
->key
, key
, keylen
);
728 ctx
->keylen
= keylen
;
733 static int omap_aes_ecb_encrypt(struct ablkcipher_request
*req
)
735 return omap_aes_crypt(req
, FLAGS_ENCRYPT
);
738 static int omap_aes_ecb_decrypt(struct ablkcipher_request
*req
)
740 return omap_aes_crypt(req
, 0);
743 static int omap_aes_cbc_encrypt(struct ablkcipher_request
*req
)
745 return omap_aes_crypt(req
, FLAGS_ENCRYPT
| FLAGS_CBC
);
748 static int omap_aes_cbc_decrypt(struct ablkcipher_request
*req
)
750 return omap_aes_crypt(req
, FLAGS_CBC
);
753 static int omap_aes_ctr_encrypt(struct ablkcipher_request
*req
)
755 return omap_aes_crypt(req
, FLAGS_ENCRYPT
| FLAGS_CTR
);
758 static int omap_aes_ctr_decrypt(struct ablkcipher_request
*req
)
760 return omap_aes_crypt(req
, FLAGS_CTR
);
763 static int omap_aes_cra_init(struct crypto_tfm
*tfm
)
765 struct omap_aes_dev
*dd
= NULL
;
768 /* Find AES device, currently picks the first device */
769 spin_lock_bh(&list_lock
);
770 list_for_each_entry(dd
, &dev_list
, list
) {
773 spin_unlock_bh(&list_lock
);
775 err
= pm_runtime_get_sync(dd
->dev
);
777 dev_err(dd
->dev
, "%s: failed to get_sync(%d)\n",
782 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct omap_aes_reqctx
);
787 static void omap_aes_cra_exit(struct crypto_tfm
*tfm
)
789 struct omap_aes_dev
*dd
= NULL
;
791 /* Find AES device, currently picks the first device */
792 spin_lock_bh(&list_lock
);
793 list_for_each_entry(dd
, &dev_list
, list
) {
796 spin_unlock_bh(&list_lock
);
798 pm_runtime_put_sync(dd
->dev
);
801 /* ********************** ALGS ************************************ */
803 static struct crypto_alg algs_ecb_cbc
[] = {
805 .cra_name
= "ecb(aes)",
806 .cra_driver_name
= "ecb-aes-omap",
808 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
809 CRYPTO_ALG_KERN_DRIVER_ONLY
|
811 .cra_blocksize
= AES_BLOCK_SIZE
,
812 .cra_ctxsize
= sizeof(struct omap_aes_ctx
),
814 .cra_type
= &crypto_ablkcipher_type
,
815 .cra_module
= THIS_MODULE
,
816 .cra_init
= omap_aes_cra_init
,
817 .cra_exit
= omap_aes_cra_exit
,
818 .cra_u
.ablkcipher
= {
819 .min_keysize
= AES_MIN_KEY_SIZE
,
820 .max_keysize
= AES_MAX_KEY_SIZE
,
821 .setkey
= omap_aes_setkey
,
822 .encrypt
= omap_aes_ecb_encrypt
,
823 .decrypt
= omap_aes_ecb_decrypt
,
827 .cra_name
= "cbc(aes)",
828 .cra_driver_name
= "cbc-aes-omap",
830 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
831 CRYPTO_ALG_KERN_DRIVER_ONLY
|
833 .cra_blocksize
= AES_BLOCK_SIZE
,
834 .cra_ctxsize
= sizeof(struct omap_aes_ctx
),
836 .cra_type
= &crypto_ablkcipher_type
,
837 .cra_module
= THIS_MODULE
,
838 .cra_init
= omap_aes_cra_init
,
839 .cra_exit
= omap_aes_cra_exit
,
840 .cra_u
.ablkcipher
= {
841 .min_keysize
= AES_MIN_KEY_SIZE
,
842 .max_keysize
= AES_MAX_KEY_SIZE
,
843 .ivsize
= AES_BLOCK_SIZE
,
844 .setkey
= omap_aes_setkey
,
845 .encrypt
= omap_aes_cbc_encrypt
,
846 .decrypt
= omap_aes_cbc_decrypt
,
851 static struct crypto_alg algs_ctr
[] = {
853 .cra_name
= "ctr(aes)",
854 .cra_driver_name
= "ctr-aes-omap",
856 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
857 CRYPTO_ALG_KERN_DRIVER_ONLY
|
859 .cra_blocksize
= AES_BLOCK_SIZE
,
860 .cra_ctxsize
= sizeof(struct omap_aes_ctx
),
862 .cra_type
= &crypto_ablkcipher_type
,
863 .cra_module
= THIS_MODULE
,
864 .cra_init
= omap_aes_cra_init
,
865 .cra_exit
= omap_aes_cra_exit
,
866 .cra_u
.ablkcipher
= {
867 .min_keysize
= AES_MIN_KEY_SIZE
,
868 .max_keysize
= AES_MAX_KEY_SIZE
,
870 .ivsize
= AES_BLOCK_SIZE
,
871 .setkey
= omap_aes_setkey
,
872 .encrypt
= omap_aes_ctr_encrypt
,
873 .decrypt
= omap_aes_ctr_decrypt
,
878 static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc
[] = {
880 .algs_list
= algs_ecb_cbc
,
881 .size
= ARRAY_SIZE(algs_ecb_cbc
),
885 static const struct omap_aes_pdata omap_aes_pdata_omap2
= {
886 .algs_info
= omap_aes_algs_info_ecb_cbc
,
887 .algs_info_size
= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc
),
888 .trigger
= omap_aes_dma_trigger_omap2
,
895 .dma_enable_in
= BIT(2),
896 .dma_enable_out
= BIT(3),
905 static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr
[] = {
907 .algs_list
= algs_ecb_cbc
,
908 .size
= ARRAY_SIZE(algs_ecb_cbc
),
911 .algs_list
= algs_ctr
,
912 .size
= ARRAY_SIZE(algs_ctr
),
916 static const struct omap_aes_pdata omap_aes_pdata_omap3
= {
917 .algs_info
= omap_aes_algs_info_ecb_cbc_ctr
,
918 .algs_info_size
= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr
),
919 .trigger
= omap_aes_dma_trigger_omap2
,
926 .dma_enable_in
= BIT(2),
927 .dma_enable_out
= BIT(3),
935 static const struct omap_aes_pdata omap_aes_pdata_omap4
= {
936 .algs_info
= omap_aes_algs_info_ecb_cbc_ctr
,
937 .algs_info_size
= ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr
),
938 .trigger
= omap_aes_dma_trigger_omap4
,
945 .irq_status_ofs
= 0x8c,
946 .irq_enable_ofs
= 0x90,
947 .dma_enable_in
= BIT(5),
948 .dma_enable_out
= BIT(6),
949 .major_mask
= 0x0700,
951 .minor_mask
= 0x003f,
955 static irqreturn_t
omap_aes_irq(int irq
, void *dev_id
)
957 struct omap_aes_dev
*dd
= dev_id
;
961 status
= omap_aes_read(dd
, AES_REG_IRQ_STATUS(dd
));
962 if (status
& AES_REG_IRQ_DATA_IN
) {
963 omap_aes_write(dd
, AES_REG_IRQ_ENABLE(dd
), 0x0);
967 BUG_ON(_calc_walked(in
) > dd
->in_sg
->length
);
969 src
= sg_virt(dd
->in_sg
) + _calc_walked(in
);
971 for (i
= 0; i
< AES_BLOCK_WORDS
; i
++) {
972 omap_aes_write(dd
, AES_REG_DATA_N(dd
, i
), *src
);
974 scatterwalk_advance(&dd
->in_walk
, 4);
975 if (dd
->in_sg
->length
== _calc_walked(in
)) {
976 dd
->in_sg
= sg_next(dd
->in_sg
);
978 scatterwalk_start(&dd
->in_walk
,
980 src
= sg_virt(dd
->in_sg
) +
988 /* Clear IRQ status */
989 status
&= ~AES_REG_IRQ_DATA_IN
;
990 omap_aes_write(dd
, AES_REG_IRQ_STATUS(dd
), status
);
992 /* Enable DATA_OUT interrupt */
993 omap_aes_write(dd
, AES_REG_IRQ_ENABLE(dd
), 0x4);
995 } else if (status
& AES_REG_IRQ_DATA_OUT
) {
996 omap_aes_write(dd
, AES_REG_IRQ_ENABLE(dd
), 0x0);
1000 BUG_ON(_calc_walked(out
) > dd
->out_sg
->length
);
1002 dst
= sg_virt(dd
->out_sg
) + _calc_walked(out
);
1004 for (i
= 0; i
< AES_BLOCK_WORDS
; i
++) {
1005 *dst
= omap_aes_read(dd
, AES_REG_DATA_N(dd
, i
));
1006 scatterwalk_advance(&dd
->out_walk
, 4);
1007 if (dd
->out_sg
->length
== _calc_walked(out
)) {
1008 dd
->out_sg
= sg_next(dd
->out_sg
);
1010 scatterwalk_start(&dd
->out_walk
,
1012 dst
= sg_virt(dd
->out_sg
) +
1020 dd
->total
-= min_t(size_t, AES_BLOCK_SIZE
, dd
->total
);
1022 /* Clear IRQ status */
1023 status
&= ~AES_REG_IRQ_DATA_OUT
;
1024 omap_aes_write(dd
, AES_REG_IRQ_STATUS(dd
), status
);
1027 /* All bytes read! */
1028 tasklet_schedule(&dd
->done_task
);
1030 /* Enable DATA_IN interrupt for next block */
1031 omap_aes_write(dd
, AES_REG_IRQ_ENABLE(dd
), 0x2);
1037 static const struct of_device_id omap_aes_of_match
[] = {
1039 .compatible
= "ti,omap2-aes",
1040 .data
= &omap_aes_pdata_omap2
,
1043 .compatible
= "ti,omap3-aes",
1044 .data
= &omap_aes_pdata_omap3
,
1047 .compatible
= "ti,omap4-aes",
1048 .data
= &omap_aes_pdata_omap4
,
1052 MODULE_DEVICE_TABLE(of
, omap_aes_of_match
);
1054 static int omap_aes_get_res_of(struct omap_aes_dev
*dd
,
1055 struct device
*dev
, struct resource
*res
)
1057 struct device_node
*node
= dev
->of_node
;
1058 const struct of_device_id
*match
;
1061 match
= of_match_device(of_match_ptr(omap_aes_of_match
), dev
);
1063 dev_err(dev
, "no compatible OF match\n");
1068 err
= of_address_to_resource(node
, 0, res
);
1070 dev_err(dev
, "can't translate OF node address\n");
1075 dd
->pdata
= match
->data
;
1081 static const struct of_device_id omap_aes_of_match
[] = {
1085 static int omap_aes_get_res_of(struct omap_aes_dev
*dd
,
1086 struct device
*dev
, struct resource
*res
)
1092 static int omap_aes_get_res_pdev(struct omap_aes_dev
*dd
,
1093 struct platform_device
*pdev
, struct resource
*res
)
1095 struct device
*dev
= &pdev
->dev
;
1099 /* Get the base address */
1100 r
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1102 dev_err(dev
, "no MEM resource info\n");
1106 memcpy(res
, r
, sizeof(*res
));
1108 /* Only OMAP2/3 can be non-DT */
1109 dd
->pdata
= &omap_aes_pdata_omap2
;
1115 static int omap_aes_probe(struct platform_device
*pdev
)
1117 struct device
*dev
= &pdev
->dev
;
1118 struct omap_aes_dev
*dd
;
1119 struct crypto_alg
*algp
;
1120 struct resource res
;
1121 int err
= -ENOMEM
, i
, j
, irq
= -1;
1124 dd
= devm_kzalloc(dev
, sizeof(struct omap_aes_dev
), GFP_KERNEL
);
1126 dev_err(dev
, "unable to alloc data struct.\n");
1130 platform_set_drvdata(pdev
, dd
);
1132 err
= (dev
->of_node
) ? omap_aes_get_res_of(dd
, dev
, &res
) :
1133 omap_aes_get_res_pdev(dd
, pdev
, &res
);
1137 dd
->io_base
= devm_ioremap_resource(dev
, &res
);
1138 if (IS_ERR(dd
->io_base
)) {
1139 err
= PTR_ERR(dd
->io_base
);
1142 dd
->phys_base
= res
.start
;
1144 pm_runtime_enable(dev
);
1145 err
= pm_runtime_get_sync(dev
);
1147 dev_err(dev
, "%s: failed to get_sync(%d)\n",
1152 omap_aes_dma_stop(dd
);
1154 reg
= omap_aes_read(dd
, AES_REG_REV(dd
));
1156 pm_runtime_put_sync(dev
);
1158 dev_info(dev
, "OMAP AES hw accel rev: %u.%u\n",
1159 (reg
& dd
->pdata
->major_mask
) >> dd
->pdata
->major_shift
,
1160 (reg
& dd
->pdata
->minor_mask
) >> dd
->pdata
->minor_shift
);
1162 tasklet_init(&dd
->done_task
, omap_aes_done_task
, (unsigned long)dd
);
1164 err
= omap_aes_dma_init(dd
);
1165 if (err
== -EPROBE_DEFER
) {
1167 } else if (err
&& AES_REG_IRQ_STATUS(dd
) && AES_REG_IRQ_ENABLE(dd
)) {
1170 irq
= platform_get_irq(pdev
, 0);
1172 dev_err(dev
, "can't get IRQ resource\n");
1176 err
= devm_request_irq(dev
, irq
, omap_aes_irq
, 0,
1179 dev_err(dev
, "Unable to grab omap-aes IRQ\n");
1185 INIT_LIST_HEAD(&dd
->list
);
1186 spin_lock(&list_lock
);
1187 list_add_tail(&dd
->list
, &dev_list
);
1188 spin_unlock(&list_lock
);
1190 for (i
= 0; i
< dd
->pdata
->algs_info_size
; i
++) {
1191 if (!dd
->pdata
->algs_info
[i
].registered
) {
1192 for (j
= 0; j
< dd
->pdata
->algs_info
[i
].size
; j
++) {
1193 algp
= &dd
->pdata
->algs_info
[i
].algs_list
[j
];
1195 pr_debug("reg alg: %s\n", algp
->cra_name
);
1196 INIT_LIST_HEAD(&algp
->cra_list
);
1198 err
= crypto_register_alg(algp
);
1202 dd
->pdata
->algs_info
[i
].registered
++;
1207 /* Initialize crypto engine */
1208 dd
->engine
= crypto_engine_alloc_init(dev
, 1);
1212 dd
->engine
->prepare_cipher_request
= omap_aes_prepare_req
;
1213 dd
->engine
->cipher_one_request
= omap_aes_crypt_req
;
1214 err
= crypto_engine_start(dd
->engine
);
1220 crypto_engine_exit(dd
->engine
);
1222 for (i
= dd
->pdata
->algs_info_size
- 1; i
>= 0; i
--)
1223 for (j
= dd
->pdata
->algs_info
[i
].registered
- 1; j
>= 0; j
--)
1224 crypto_unregister_alg(
1225 &dd
->pdata
->algs_info
[i
].algs_list
[j
]);
1227 omap_aes_dma_cleanup(dd
);
1229 tasklet_kill(&dd
->done_task
);
1230 pm_runtime_disable(dev
);
1234 dev_err(dev
, "initialization failed.\n");
1238 static int omap_aes_remove(struct platform_device
*pdev
)
1240 struct omap_aes_dev
*dd
= platform_get_drvdata(pdev
);
1246 spin_lock(&list_lock
);
1247 list_del(&dd
->list
);
1248 spin_unlock(&list_lock
);
1250 for (i
= dd
->pdata
->algs_info_size
- 1; i
>= 0; i
--)
1251 for (j
= dd
->pdata
->algs_info
[i
].registered
- 1; j
>= 0; j
--)
1252 crypto_unregister_alg(
1253 &dd
->pdata
->algs_info
[i
].algs_list
[j
]);
1255 crypto_engine_exit(dd
->engine
);
1256 tasklet_kill(&dd
->done_task
);
1257 omap_aes_dma_cleanup(dd
);
1258 pm_runtime_disable(dd
->dev
);
1264 #ifdef CONFIG_PM_SLEEP
1265 static int omap_aes_suspend(struct device
*dev
)
1267 pm_runtime_put_sync(dev
);
1271 static int omap_aes_resume(struct device
*dev
)
1273 pm_runtime_get_sync(dev
);
1278 static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops
, omap_aes_suspend
, omap_aes_resume
);
1280 static struct platform_driver omap_aes_driver
= {
1281 .probe
= omap_aes_probe
,
1282 .remove
= omap_aes_remove
,
1285 .pm
= &omap_aes_pm_ops
,
1286 .of_match_table
= omap_aes_of_match
,
1290 module_platform_driver(omap_aes_driver
);
1292 MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
1293 MODULE_LICENSE("GPL v2");
1294 MODULE_AUTHOR("Dmitry Kasatkin");