2 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 * Copyright (c) 2008 Freescale Semiconductor, Inc.
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_platform.h>
36 #include <linux/dma-mapping.h>
38 #include <linux/spinlock.h>
39 #include <linux/rtnetlink.h>
40 #include <linux/slab.h>
42 #include <crypto/algapi.h>
43 #include <crypto/aes.h>
44 #include <crypto/des.h>
45 #include <crypto/sha.h>
46 #include <crypto/aead.h>
47 #include <crypto/authenc.h>
48 #include <crypto/skcipher.h>
49 #include <crypto/scatterwalk.h>
53 #define TALITOS_TIMEOUT 100000
54 #define TALITOS_MAX_DATA_LEN 65535
56 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
57 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
58 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
60 /* descriptor pointer entry */
62 __be16 len
; /* length */
63 u8 j_extent
; /* jump to sg link table and/or extent */
64 u8 eptr
; /* extended address */
65 __be32 ptr
; /* address */
70 __be32 hdr
; /* header high bits */
71 __be32 hdr_lo
; /* header low bits */
72 struct talitos_ptr ptr
[7]; /* ptr/len pair array */
76 * talitos_request - descriptor submission request
77 * @desc: descriptor pointer (kernel virtual)
78 * @dma_desc: descriptor's physical bus address
79 * @callback: whom to call when descriptor processing is done
80 * @context: caller context (optional)
82 struct talitos_request
{
83 struct talitos_desc
*desc
;
85 void (*callback
) (struct device
*dev
, struct talitos_desc
*desc
,
86 void *context
, int error
);
90 /* per-channel fifo management */
91 struct talitos_channel
{
93 struct talitos_request
*fifo
;
95 /* number of requests pending in channel h/w fifo */
96 atomic_t submit_count ____cacheline_aligned
;
98 /* request submission (head) lock */
99 spinlock_t head_lock ____cacheline_aligned
;
100 /* index to next free descriptor request */
103 /* request release (tail) lock */
104 spinlock_t tail_lock ____cacheline_aligned
;
105 /* index to next in-progress/done descriptor request */
109 struct talitos_private
{
111 struct of_device
*ofdev
;
115 /* SEC version geometry (from device tree node) */
116 unsigned int num_channels
;
117 unsigned int chfifo_len
;
118 unsigned int exec_units
;
119 unsigned int desc_types
;
121 /* SEC Compatibility info */
122 unsigned long features
;
125 * length of the request fifo
126 * fifo_len is chfifo_len rounded up to next power of 2
127 * so we can use bitwise ops to wrap
129 unsigned int fifo_len
;
131 struct talitos_channel
*chan
;
133 /* next channel to be assigned next incoming descriptor */
134 atomic_t last_chan ____cacheline_aligned
;
136 /* request callback tasklet */
137 struct tasklet_struct done_task
;
139 /* list of registered algorithms */
140 struct list_head alg_list
;
147 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
148 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
150 static void to_talitos_ptr(struct talitos_ptr
*talitos_ptr
, dma_addr_t dma_addr
)
152 talitos_ptr
->ptr
= cpu_to_be32(lower_32_bits(dma_addr
));
153 talitos_ptr
->eptr
= cpu_to_be32(upper_32_bits(dma_addr
));
157 * map virtual single (contiguous) pointer to h/w descriptor pointer
159 static void map_single_talitos_ptr(struct device
*dev
,
160 struct talitos_ptr
*talitos_ptr
,
161 unsigned short len
, void *data
,
162 unsigned char extent
,
163 enum dma_data_direction dir
)
165 dma_addr_t dma_addr
= dma_map_single(dev
, data
, len
, dir
);
167 talitos_ptr
->len
= cpu_to_be16(len
);
168 to_talitos_ptr(talitos_ptr
, dma_addr
);
169 talitos_ptr
->j_extent
= extent
;
173 * unmap bus single (contiguous) h/w descriptor pointer
175 static void unmap_single_talitos_ptr(struct device
*dev
,
176 struct talitos_ptr
*talitos_ptr
,
177 enum dma_data_direction dir
)
179 dma_unmap_single(dev
, be32_to_cpu(talitos_ptr
->ptr
),
180 be16_to_cpu(talitos_ptr
->len
), dir
);
183 static int reset_channel(struct device
*dev
, int ch
)
185 struct talitos_private
*priv
= dev_get_drvdata(dev
);
186 unsigned int timeout
= TALITOS_TIMEOUT
;
188 setbits32(priv
->reg
+ TALITOS_CCCR(ch
), TALITOS_CCCR_RESET
);
190 while ((in_be32(priv
->reg
+ TALITOS_CCCR(ch
)) & TALITOS_CCCR_RESET
)
195 dev_err(dev
, "failed to reset channel %d\n", ch
);
199 /* set 36-bit addressing, done writeback enable and done IRQ enable */
200 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
), TALITOS_CCCR_LO_EAE
|
201 TALITOS_CCCR_LO_CDWE
| TALITOS_CCCR_LO_CDIE
);
203 /* and ICCR writeback, if available */
204 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
205 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
),
206 TALITOS_CCCR_LO_IWSE
);
211 static int reset_device(struct device
*dev
)
213 struct talitos_private
*priv
= dev_get_drvdata(dev
);
214 unsigned int timeout
= TALITOS_TIMEOUT
;
216 setbits32(priv
->reg
+ TALITOS_MCR
, TALITOS_MCR_SWR
);
218 while ((in_be32(priv
->reg
+ TALITOS_MCR
) & TALITOS_MCR_SWR
)
223 dev_err(dev
, "failed to reset device\n");
231 * Reset and initialize the device
233 static int init_device(struct device
*dev
)
235 struct talitos_private
*priv
= dev_get_drvdata(dev
);
240 * errata documentation: warning: certain SEC interrupts
241 * are not fully cleared by writing the MCR:SWR bit,
242 * set bit twice to completely reset
244 err
= reset_device(dev
);
248 err
= reset_device(dev
);
253 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
254 err
= reset_channel(dev
, ch
);
259 /* enable channel done and error interrupts */
260 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_INIT
);
261 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS_IMR_LO_INIT
);
263 /* disable integrity check error interrupts (use writeback instead) */
264 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
265 setbits32(priv
->reg
+ TALITOS_MDEUICR_LO
,
266 TALITOS_MDEUICR_LO_ICE
);
272 * talitos_submit - submits a descriptor to the device for processing
273 * @dev: the SEC device to be used
274 * @desc: the descriptor to be processed by the device
275 * @callback: whom to call when processing is complete
276 * @context: a handle for use by caller (optional)
278 * desc must contain valid dma-mapped (bus physical) address pointers.
279 * callback must check err and feedback in descriptor header
280 * for device processing status.
282 static int talitos_submit(struct device
*dev
, struct talitos_desc
*desc
,
283 void (*callback
)(struct device
*dev
,
284 struct talitos_desc
*desc
,
285 void *context
, int error
),
288 struct talitos_private
*priv
= dev_get_drvdata(dev
);
289 struct talitos_request
*request
;
290 unsigned long flags
, ch
;
293 /* select done notification */
294 desc
->hdr
|= DESC_HDR_DONE_NOTIFY
;
296 /* emulate SEC's round-robin channel fifo polling scheme */
297 ch
= atomic_inc_return(&priv
->last_chan
) & (priv
->num_channels
- 1);
299 spin_lock_irqsave(&priv
->chan
[ch
].head_lock
, flags
);
301 if (!atomic_inc_not_zero(&priv
->chan
[ch
].submit_count
)) {
302 /* h/w fifo is full */
303 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
307 head
= priv
->chan
[ch
].head
;
308 request
= &priv
->chan
[ch
].fifo
[head
];
310 /* map descriptor and save caller data */
311 request
->dma_desc
= dma_map_single(dev
, desc
, sizeof(*desc
),
313 request
->callback
= callback
;
314 request
->context
= context
;
316 /* increment fifo head */
317 priv
->chan
[ch
].head
= (priv
->chan
[ch
].head
+ 1) & (priv
->fifo_len
- 1);
320 request
->desc
= desc
;
324 out_be32(priv
->reg
+ TALITOS_FF(ch
),
325 cpu_to_be32(upper_32_bits(request
->dma_desc
)));
326 out_be32(priv
->reg
+ TALITOS_FF_LO(ch
),
327 cpu_to_be32(lower_32_bits(request
->dma_desc
)));
329 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
335 * process what was done, notify callback of error if not
337 static void flush_channel(struct device
*dev
, int ch
, int error
, int reset_ch
)
339 struct talitos_private
*priv
= dev_get_drvdata(dev
);
340 struct talitos_request
*request
, saved_req
;
344 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
346 tail
= priv
->chan
[ch
].tail
;
347 while (priv
->chan
[ch
].fifo
[tail
].desc
) {
348 request
= &priv
->chan
[ch
].fifo
[tail
];
350 /* descriptors with their done bits set don't get the error */
352 if ((request
->desc
->hdr
& DESC_HDR_DONE
) == DESC_HDR_DONE
)
360 dma_unmap_single(dev
, request
->dma_desc
,
361 sizeof(struct talitos_desc
),
364 /* copy entries so we can call callback outside lock */
365 saved_req
.desc
= request
->desc
;
366 saved_req
.callback
= request
->callback
;
367 saved_req
.context
= request
->context
;
369 /* release request entry in fifo */
371 request
->desc
= NULL
;
373 /* increment fifo tail */
374 priv
->chan
[ch
].tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
376 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
378 atomic_dec(&priv
->chan
[ch
].submit_count
);
380 saved_req
.callback(dev
, saved_req
.desc
, saved_req
.context
,
382 /* channel may resume processing in single desc error case */
383 if (error
&& !reset_ch
&& status
== error
)
385 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
386 tail
= priv
->chan
[ch
].tail
;
389 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
393 * process completed requests for channels that have done status
395 static void talitos_done(unsigned long data
)
397 struct device
*dev
= (struct device
*)data
;
398 struct talitos_private
*priv
= dev_get_drvdata(dev
);
401 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
402 flush_channel(dev
, ch
, 0, 0);
404 /* At this point, all completed channels have been processed.
405 * Unmask done interrupts for channels completed later on.
407 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_INIT
);
408 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS_IMR_LO_INIT
);
412 * locate current (offending) descriptor
414 static struct talitos_desc
*current_desc(struct device
*dev
, int ch
)
416 struct talitos_private
*priv
= dev_get_drvdata(dev
);
417 int tail
= priv
->chan
[ch
].tail
;
420 cur_desc
= in_be32(priv
->reg
+ TALITOS_CDPR_LO(ch
));
422 while (priv
->chan
[ch
].fifo
[tail
].dma_desc
!= cur_desc
) {
423 tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
424 if (tail
== priv
->chan
[ch
].tail
) {
425 dev_err(dev
, "couldn't locate current descriptor\n");
430 return priv
->chan
[ch
].fifo
[tail
].desc
;
434 * user diagnostics; report root cause of error based on execution unit status
436 static void report_eu_error(struct device
*dev
, int ch
,
437 struct talitos_desc
*desc
)
439 struct talitos_private
*priv
= dev_get_drvdata(dev
);
442 switch (desc
->hdr
& DESC_HDR_SEL0_MASK
) {
443 case DESC_HDR_SEL0_AFEU
:
444 dev_err(dev
, "AFEUISR 0x%08x_%08x\n",
445 in_be32(priv
->reg
+ TALITOS_AFEUISR
),
446 in_be32(priv
->reg
+ TALITOS_AFEUISR_LO
));
448 case DESC_HDR_SEL0_DEU
:
449 dev_err(dev
, "DEUISR 0x%08x_%08x\n",
450 in_be32(priv
->reg
+ TALITOS_DEUISR
),
451 in_be32(priv
->reg
+ TALITOS_DEUISR_LO
));
453 case DESC_HDR_SEL0_MDEUA
:
454 case DESC_HDR_SEL0_MDEUB
:
455 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
456 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
457 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
459 case DESC_HDR_SEL0_RNG
:
460 dev_err(dev
, "RNGUISR 0x%08x_%08x\n",
461 in_be32(priv
->reg
+ TALITOS_RNGUISR
),
462 in_be32(priv
->reg
+ TALITOS_RNGUISR_LO
));
464 case DESC_HDR_SEL0_PKEU
:
465 dev_err(dev
, "PKEUISR 0x%08x_%08x\n",
466 in_be32(priv
->reg
+ TALITOS_PKEUISR
),
467 in_be32(priv
->reg
+ TALITOS_PKEUISR_LO
));
469 case DESC_HDR_SEL0_AESU
:
470 dev_err(dev
, "AESUISR 0x%08x_%08x\n",
471 in_be32(priv
->reg
+ TALITOS_AESUISR
),
472 in_be32(priv
->reg
+ TALITOS_AESUISR_LO
));
474 case DESC_HDR_SEL0_CRCU
:
475 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
476 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
477 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
479 case DESC_HDR_SEL0_KEU
:
480 dev_err(dev
, "KEUISR 0x%08x_%08x\n",
481 in_be32(priv
->reg
+ TALITOS_KEUISR
),
482 in_be32(priv
->reg
+ TALITOS_KEUISR_LO
));
486 switch (desc
->hdr
& DESC_HDR_SEL1_MASK
) {
487 case DESC_HDR_SEL1_MDEUA
:
488 case DESC_HDR_SEL1_MDEUB
:
489 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
490 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
491 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
493 case DESC_HDR_SEL1_CRCU
:
494 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
495 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
496 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
500 for (i
= 0; i
< 8; i
++)
501 dev_err(dev
, "DESCBUF 0x%08x_%08x\n",
502 in_be32(priv
->reg
+ TALITOS_DESCBUF(ch
) + 8*i
),
503 in_be32(priv
->reg
+ TALITOS_DESCBUF_LO(ch
) + 8*i
));
507 * recover from error interrupts
509 static void talitos_error(unsigned long data
, u32 isr
, u32 isr_lo
)
511 struct device
*dev
= (struct device
*)data
;
512 struct talitos_private
*priv
= dev_get_drvdata(dev
);
513 unsigned int timeout
= TALITOS_TIMEOUT
;
514 int ch
, error
, reset_dev
= 0, reset_ch
= 0;
517 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
518 /* skip channels without errors */
519 if (!(isr
& (1 << (ch
* 2 + 1))))
524 v
= in_be32(priv
->reg
+ TALITOS_CCPSR(ch
));
525 v_lo
= in_be32(priv
->reg
+ TALITOS_CCPSR_LO(ch
));
527 if (v_lo
& TALITOS_CCPSR_LO_DOF
) {
528 dev_err(dev
, "double fetch fifo overflow error\n");
532 if (v_lo
& TALITOS_CCPSR_LO_SOF
) {
533 /* h/w dropped descriptor */
534 dev_err(dev
, "single fetch fifo overflow error\n");
537 if (v_lo
& TALITOS_CCPSR_LO_MDTE
)
538 dev_err(dev
, "master data transfer error\n");
539 if (v_lo
& TALITOS_CCPSR_LO_SGDLZ
)
540 dev_err(dev
, "s/g data length zero error\n");
541 if (v_lo
& TALITOS_CCPSR_LO_FPZ
)
542 dev_err(dev
, "fetch pointer zero error\n");
543 if (v_lo
& TALITOS_CCPSR_LO_IDH
)
544 dev_err(dev
, "illegal descriptor header error\n");
545 if (v_lo
& TALITOS_CCPSR_LO_IEU
)
546 dev_err(dev
, "invalid execution unit error\n");
547 if (v_lo
& TALITOS_CCPSR_LO_EU
)
548 report_eu_error(dev
, ch
, current_desc(dev
, ch
));
549 if (v_lo
& TALITOS_CCPSR_LO_GB
)
550 dev_err(dev
, "gather boundary error\n");
551 if (v_lo
& TALITOS_CCPSR_LO_GRL
)
552 dev_err(dev
, "gather return/length error\n");
553 if (v_lo
& TALITOS_CCPSR_LO_SB
)
554 dev_err(dev
, "scatter boundary error\n");
555 if (v_lo
& TALITOS_CCPSR_LO_SRL
)
556 dev_err(dev
, "scatter return/length error\n");
558 flush_channel(dev
, ch
, error
, reset_ch
);
561 reset_channel(dev
, ch
);
563 setbits32(priv
->reg
+ TALITOS_CCCR(ch
),
565 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
), 0);
566 while ((in_be32(priv
->reg
+ TALITOS_CCCR(ch
)) &
567 TALITOS_CCCR_CONT
) && --timeout
)
570 dev_err(dev
, "failed to restart channel %d\n",
576 if (reset_dev
|| isr
& ~TALITOS_ISR_CHERR
|| isr_lo
) {
577 dev_err(dev
, "done overflow, internal time out, or rngu error: "
578 "ISR 0x%08x_%08x\n", isr
, isr_lo
);
580 /* purge request queues */
581 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
582 flush_channel(dev
, ch
, -EIO
, 1);
584 /* reset and reinitialize the device */
589 static irqreturn_t
talitos_interrupt(int irq
, void *data
)
591 struct device
*dev
= data
;
592 struct talitos_private
*priv
= dev_get_drvdata(dev
);
595 isr
= in_be32(priv
->reg
+ TALITOS_ISR
);
596 isr_lo
= in_be32(priv
->reg
+ TALITOS_ISR_LO
);
597 /* Acknowledge interrupt */
598 out_be32(priv
->reg
+ TALITOS_ICR
, isr
);
599 out_be32(priv
->reg
+ TALITOS_ICR_LO
, isr_lo
);
601 if (unlikely((isr
& ~TALITOS_ISR_CHDONE
) || isr_lo
))
602 talitos_error((unsigned long)data
, isr
, isr_lo
);
604 if (likely(isr
& TALITOS_ISR_CHDONE
)) {
605 /* mask further done interrupts. */
606 clrbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_DONE
);
607 /* done_task will unmask done interrupts at exit */
608 tasklet_schedule(&priv
->done_task
);
611 return (isr
|| isr_lo
) ? IRQ_HANDLED
: IRQ_NONE
;
617 static int talitos_rng_data_present(struct hwrng
*rng
, int wait
)
619 struct device
*dev
= (struct device
*)rng
->priv
;
620 struct talitos_private
*priv
= dev_get_drvdata(dev
);
624 for (i
= 0; i
< 20; i
++) {
625 ofl
= in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) &
626 TALITOS_RNGUSR_LO_OFL
;
635 static int talitos_rng_data_read(struct hwrng
*rng
, u32
*data
)
637 struct device
*dev
= (struct device
*)rng
->priv
;
638 struct talitos_private
*priv
= dev_get_drvdata(dev
);
640 /* rng fifo requires 64-bit accesses */
641 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO
);
642 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO_LO
);
647 static int talitos_rng_init(struct hwrng
*rng
)
649 struct device
*dev
= (struct device
*)rng
->priv
;
650 struct talitos_private
*priv
= dev_get_drvdata(dev
);
651 unsigned int timeout
= TALITOS_TIMEOUT
;
653 setbits32(priv
->reg
+ TALITOS_RNGURCR_LO
, TALITOS_RNGURCR_LO_SR
);
654 while (!(in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) & TALITOS_RNGUSR_LO_RD
)
658 dev_err(dev
, "failed to reset rng hw\n");
662 /* start generating */
663 setbits32(priv
->reg
+ TALITOS_RNGUDSR_LO
, 0);
668 static int talitos_register_rng(struct device
*dev
)
670 struct talitos_private
*priv
= dev_get_drvdata(dev
);
672 priv
->rng
.name
= dev_driver_string(dev
),
673 priv
->rng
.init
= talitos_rng_init
,
674 priv
->rng
.data_present
= talitos_rng_data_present
,
675 priv
->rng
.data_read
= talitos_rng_data_read
,
676 priv
->rng
.priv
= (unsigned long)dev
;
678 return hwrng_register(&priv
->rng
);
681 static void talitos_unregister_rng(struct device
*dev
)
683 struct talitos_private
*priv
= dev_get_drvdata(dev
);
685 hwrng_unregister(&priv
->rng
);
691 #define TALITOS_CRA_PRIORITY 3000
692 #define TALITOS_MAX_KEY_SIZE 64
693 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
695 #define MD5_DIGEST_SIZE 16
699 __be32 desc_hdr_template
;
700 u8 key
[TALITOS_MAX_KEY_SIZE
];
701 u8 iv
[TALITOS_MAX_IV_LENGTH
];
703 unsigned int enckeylen
;
704 unsigned int authkeylen
;
705 unsigned int authsize
;
708 static int aead_setauthsize(struct crypto_aead
*authenc
,
709 unsigned int authsize
)
711 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
713 ctx
->authsize
= authsize
;
718 static int aead_setkey(struct crypto_aead
*authenc
,
719 const u8
*key
, unsigned int keylen
)
721 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
722 struct rtattr
*rta
= (void *)key
;
723 struct crypto_authenc_key_param
*param
;
724 unsigned int authkeylen
;
725 unsigned int enckeylen
;
727 if (!RTA_OK(rta
, keylen
))
730 if (rta
->rta_type
!= CRYPTO_AUTHENC_KEYA_PARAM
)
733 if (RTA_PAYLOAD(rta
) < sizeof(*param
))
736 param
= RTA_DATA(rta
);
737 enckeylen
= be32_to_cpu(param
->enckeylen
);
739 key
+= RTA_ALIGN(rta
->rta_len
);
740 keylen
-= RTA_ALIGN(rta
->rta_len
);
742 if (keylen
< enckeylen
)
745 authkeylen
= keylen
- enckeylen
;
747 if (keylen
> TALITOS_MAX_KEY_SIZE
)
750 memcpy(&ctx
->key
, key
, keylen
);
752 ctx
->keylen
= keylen
;
753 ctx
->enckeylen
= enckeylen
;
754 ctx
->authkeylen
= authkeylen
;
759 crypto_aead_set_flags(authenc
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
764 * talitos_edesc - s/w-extended descriptor
765 * @src_nents: number of segments in input scatterlist
766 * @dst_nents: number of segments in output scatterlist
767 * @dma_len: length of dma mapped link_tbl space
768 * @dma_link_tbl: bus physical address of link_tbl
769 * @desc: h/w descriptor
770 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
772 * if decrypting (with authcheck), or either one of src_nents or dst_nents
773 * is greater than 1, an integrity check value is concatenated to the end
776 struct talitos_edesc
{
782 dma_addr_t dma_link_tbl
;
783 struct talitos_desc desc
;
784 struct talitos_ptr link_tbl
[0];
787 static int talitos_map_sg(struct device
*dev
, struct scatterlist
*sg
,
788 unsigned int nents
, enum dma_data_direction dir
,
791 if (unlikely(chained
))
793 dma_map_sg(dev
, sg
, 1, dir
);
794 sg
= scatterwalk_sg_next(sg
);
797 dma_map_sg(dev
, sg
, nents
, dir
);
801 static void talitos_unmap_sg_chain(struct device
*dev
, struct scatterlist
*sg
,
802 enum dma_data_direction dir
)
805 dma_unmap_sg(dev
, sg
, 1, dir
);
806 sg
= scatterwalk_sg_next(sg
);
810 static void talitos_sg_unmap(struct device
*dev
,
811 struct talitos_edesc
*edesc
,
812 struct scatterlist
*src
,
813 struct scatterlist
*dst
)
815 unsigned int src_nents
= edesc
->src_nents
? : 1;
816 unsigned int dst_nents
= edesc
->dst_nents
? : 1;
819 if (edesc
->src_is_chained
)
820 talitos_unmap_sg_chain(dev
, src
, DMA_TO_DEVICE
);
822 dma_unmap_sg(dev
, src
, src_nents
, DMA_TO_DEVICE
);
824 if (edesc
->dst_is_chained
)
825 talitos_unmap_sg_chain(dev
, dst
, DMA_FROM_DEVICE
);
827 dma_unmap_sg(dev
, dst
, dst_nents
, DMA_FROM_DEVICE
);
829 if (edesc
->src_is_chained
)
830 talitos_unmap_sg_chain(dev
, src
, DMA_BIDIRECTIONAL
);
832 dma_unmap_sg(dev
, src
, src_nents
, DMA_BIDIRECTIONAL
);
835 static void ipsec_esp_unmap(struct device
*dev
,
836 struct talitos_edesc
*edesc
,
837 struct aead_request
*areq
)
839 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[6], DMA_FROM_DEVICE
);
840 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[3], DMA_TO_DEVICE
);
841 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
842 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[0], DMA_TO_DEVICE
);
844 dma_unmap_sg(dev
, areq
->assoc
, 1, DMA_TO_DEVICE
);
846 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
849 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
854 * ipsec_esp descriptor callbacks
856 static void ipsec_esp_encrypt_done(struct device
*dev
,
857 struct talitos_desc
*desc
, void *context
,
860 struct aead_request
*areq
= context
;
861 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
862 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
863 struct talitos_edesc
*edesc
;
864 struct scatterlist
*sg
;
867 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
869 ipsec_esp_unmap(dev
, edesc
, areq
);
871 /* copy the generated ICV to dst */
872 if (edesc
->dma_len
) {
873 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
874 edesc
->dst_nents
+ 2];
875 sg
= sg_last(areq
->dst
, edesc
->dst_nents
);
876 memcpy((char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
877 icvdata
, ctx
->authsize
);
882 aead_request_complete(areq
, err
);
885 static void ipsec_esp_decrypt_swauth_done(struct device
*dev
,
886 struct talitos_desc
*desc
,
887 void *context
, int err
)
889 struct aead_request
*req
= context
;
890 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
891 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
892 struct talitos_edesc
*edesc
;
893 struct scatterlist
*sg
;
896 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
898 ipsec_esp_unmap(dev
, edesc
, req
);
903 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
904 edesc
->dst_nents
+ 2];
906 icvdata
= &edesc
->link_tbl
[0];
908 sg
= sg_last(req
->dst
, edesc
->dst_nents
? : 1);
909 err
= memcmp(icvdata
, (char *)sg_virt(sg
) + sg
->length
-
910 ctx
->authsize
, ctx
->authsize
) ? -EBADMSG
: 0;
915 aead_request_complete(req
, err
);
918 static void ipsec_esp_decrypt_hwauth_done(struct device
*dev
,
919 struct talitos_desc
*desc
,
920 void *context
, int err
)
922 struct aead_request
*req
= context
;
923 struct talitos_edesc
*edesc
;
925 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
927 ipsec_esp_unmap(dev
, edesc
, req
);
929 /* check ICV auth status */
930 if (!err
&& ((desc
->hdr_lo
& DESC_HDR_LO_ICCR1_MASK
) !=
931 DESC_HDR_LO_ICCR1_PASS
))
936 aead_request_complete(req
, err
);
940 * convert scatterlist to SEC h/w link table format
941 * stop at cryptlen bytes
943 static int sg_to_link_tbl(struct scatterlist
*sg
, int sg_count
,
944 int cryptlen
, struct talitos_ptr
*link_tbl_ptr
)
949 to_talitos_ptr(link_tbl_ptr
, sg_dma_address(sg
));
950 link_tbl_ptr
->len
= cpu_to_be16(sg_dma_len(sg
));
951 link_tbl_ptr
->j_extent
= 0;
953 cryptlen
-= sg_dma_len(sg
);
954 sg
= scatterwalk_sg_next(sg
);
957 /* adjust (decrease) last one (or two) entry's len to cryptlen */
959 while (be16_to_cpu(link_tbl_ptr
->len
) <= (-cryptlen
)) {
960 /* Empty this entry, and move to previous one */
961 cryptlen
+= be16_to_cpu(link_tbl_ptr
->len
);
962 link_tbl_ptr
->len
= 0;
966 link_tbl_ptr
->len
= cpu_to_be16(be16_to_cpu(link_tbl_ptr
->len
)
969 /* tag end of link table */
970 link_tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
976 * fill in and submit ipsec_esp descriptor
978 static int ipsec_esp(struct talitos_edesc
*edesc
, struct aead_request
*areq
,
980 void (*callback
) (struct device
*dev
,
981 struct talitos_desc
*desc
,
982 void *context
, int error
))
984 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
985 struct talitos_ctx
*ctx
= crypto_aead_ctx(aead
);
986 struct device
*dev
= ctx
->dev
;
987 struct talitos_desc
*desc
= &edesc
->desc
;
988 unsigned int cryptlen
= areq
->cryptlen
;
989 unsigned int authsize
= ctx
->authsize
;
990 unsigned int ivsize
= crypto_aead_ivsize(aead
);
995 map_single_talitos_ptr(dev
, &desc
->ptr
[0], ctx
->authkeylen
, &ctx
->key
,
998 map_single_talitos_ptr(dev
, &desc
->ptr
[1], areq
->assoclen
+ ivsize
,
999 sg_virt(areq
->assoc
), 0, DMA_TO_DEVICE
);
1001 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ivsize
, giv
?: areq
->iv
, 0,
1005 map_single_talitos_ptr(dev
, &desc
->ptr
[3], ctx
->enckeylen
,
1006 (char *)&ctx
->key
+ ctx
->authkeylen
, 0,
1011 * map and adjust cipher len to aead request cryptlen.
1012 * extent is bytes of HMAC postpended to ciphertext,
1013 * typically 12 for ipsec
1015 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1016 desc
->ptr
[4].j_extent
= authsize
;
1018 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
? : 1,
1019 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1021 edesc
->src_is_chained
);
1023 if (sg_count
== 1) {
1024 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->src
));
1026 sg_link_tbl_len
= cryptlen
;
1028 if (edesc
->desc
.hdr
& DESC_HDR_MODE1_MDEU_CICV
)
1029 sg_link_tbl_len
= cryptlen
+ authsize
;
1031 sg_count
= sg_to_link_tbl(areq
->src
, sg_count
, sg_link_tbl_len
,
1032 &edesc
->link_tbl
[0]);
1034 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1035 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
);
1036 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1040 /* Only one segment now, so no link tbl needed */
1041 to_talitos_ptr(&desc
->ptr
[4],
1042 sg_dma_address(areq
->src
));
1047 desc
->ptr
[5].len
= cpu_to_be16(cryptlen
);
1048 desc
->ptr
[5].j_extent
= authsize
;
1050 if (areq
->src
!= areq
->dst
)
1051 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1052 edesc
->dst_nents
? : 1,
1054 edesc
->dst_is_chained
);
1056 if (sg_count
== 1) {
1057 to_talitos_ptr(&desc
->ptr
[5], sg_dma_address(areq
->dst
));
1059 struct talitos_ptr
*link_tbl_ptr
=
1060 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1062 to_talitos_ptr(&desc
->ptr
[5], edesc
->dma_link_tbl
+
1063 (edesc
->src_nents
+ 1) *
1064 sizeof(struct talitos_ptr
));
1065 sg_count
= sg_to_link_tbl(areq
->dst
, sg_count
, cryptlen
,
1068 /* Add an entry to the link table for ICV data */
1069 link_tbl_ptr
+= sg_count
- 1;
1070 link_tbl_ptr
->j_extent
= 0;
1073 link_tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1074 link_tbl_ptr
->len
= cpu_to_be16(authsize
);
1076 /* icv data follows link tables */
1077 to_talitos_ptr(link_tbl_ptr
, edesc
->dma_link_tbl
+
1078 (edesc
->src_nents
+ edesc
->dst_nents
+ 2) *
1079 sizeof(struct talitos_ptr
));
1080 desc
->ptr
[5].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1081 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1082 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1086 map_single_talitos_ptr(dev
, &desc
->ptr
[6], ivsize
, ctx
->iv
, 0,
1089 ret
= talitos_submit(dev
, desc
, callback
, areq
);
1090 if (ret
!= -EINPROGRESS
) {
1091 ipsec_esp_unmap(dev
, edesc
, areq
);
1098 * derive number of elements in scatterlist
1100 static int sg_count(struct scatterlist
*sg_list
, int nbytes
, int *chained
)
1102 struct scatterlist
*sg
= sg_list
;
1106 while (nbytes
> 0) {
1108 nbytes
-= sg
->length
;
1109 if (!sg_is_last(sg
) && (sg
+ 1)->length
== 0)
1111 sg
= scatterwalk_sg_next(sg
);
1118 * allocate and map the extended descriptor
1120 static struct talitos_edesc
*talitos_edesc_alloc(struct device
*dev
,
1121 struct scatterlist
*src
,
1122 struct scatterlist
*dst
,
1123 unsigned int cryptlen
,
1124 unsigned int authsize
,
1128 struct talitos_edesc
*edesc
;
1129 int src_nents
, dst_nents
, alloc_len
, dma_len
;
1130 int src_chained
, dst_chained
= 0;
1131 gfp_t flags
= cryptoflags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
1134 if (cryptlen
+ authsize
> TALITOS_MAX_DATA_LEN
) {
1135 dev_err(dev
, "length exceeds h/w max limit\n");
1136 return ERR_PTR(-EINVAL
);
1139 src_nents
= sg_count(src
, cryptlen
+ authsize
, &src_chained
);
1140 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1143 dst_nents
= src_nents
;
1145 dst_nents
= sg_count(dst
, cryptlen
+ authsize
, &dst_chained
);
1146 dst_nents
= (dst_nents
== 1) ? 0 : dst_nents
;
1150 * allocate space for base edesc plus the link tables,
1151 * allowing for two separate entries for ICV and generated ICV (+ 2),
1152 * and the ICV data itself
1154 alloc_len
= sizeof(struct talitos_edesc
);
1155 if (src_nents
|| dst_nents
) {
1156 dma_len
= (src_nents
+ dst_nents
+ 2) *
1157 sizeof(struct talitos_ptr
) + authsize
;
1158 alloc_len
+= dma_len
;
1161 alloc_len
+= icv_stashing
? authsize
: 0;
1164 edesc
= kmalloc(alloc_len
, GFP_DMA
| flags
);
1166 dev_err(dev
, "could not allocate edescriptor\n");
1167 return ERR_PTR(-ENOMEM
);
1170 edesc
->src_nents
= src_nents
;
1171 edesc
->dst_nents
= dst_nents
;
1172 edesc
->src_is_chained
= src_chained
;
1173 edesc
->dst_is_chained
= dst_chained
;
1174 edesc
->dma_len
= dma_len
;
1175 edesc
->dma_link_tbl
= dma_map_single(dev
, &edesc
->link_tbl
[0],
1176 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1181 static struct talitos_edesc
*aead_edesc_alloc(struct aead_request
*areq
,
1184 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1185 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1187 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
,
1188 areq
->cryptlen
, ctx
->authsize
, icv_stashing
,
1192 static int aead_encrypt(struct aead_request
*req
)
1194 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1195 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1196 struct talitos_edesc
*edesc
;
1198 /* allocate extended descriptor */
1199 edesc
= aead_edesc_alloc(req
, 0);
1201 return PTR_ERR(edesc
);
1204 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1206 return ipsec_esp(edesc
, req
, NULL
, 0, ipsec_esp_encrypt_done
);
1209 static int aead_decrypt(struct aead_request
*req
)
1211 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1212 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1213 unsigned int authsize
= ctx
->authsize
;
1214 struct talitos_private
*priv
= dev_get_drvdata(ctx
->dev
);
1215 struct talitos_edesc
*edesc
;
1216 struct scatterlist
*sg
;
1219 req
->cryptlen
-= authsize
;
1221 /* allocate extended descriptor */
1222 edesc
= aead_edesc_alloc(req
, 1);
1224 return PTR_ERR(edesc
);
1226 if ((priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
) &&
1227 ((!edesc
->src_nents
&& !edesc
->dst_nents
) ||
1228 priv
->features
& TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
)) {
1230 /* decrypt and check the ICV */
1231 edesc
->desc
.hdr
= ctx
->desc_hdr_template
|
1232 DESC_HDR_DIR_INBOUND
|
1233 DESC_HDR_MODE1_MDEU_CICV
;
1235 /* reset integrity check result bits */
1236 edesc
->desc
.hdr_lo
= 0;
1238 return ipsec_esp(edesc
, req
, NULL
, 0,
1239 ipsec_esp_decrypt_hwauth_done
);
1243 /* Have to check the ICV with software */
1244 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1246 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1248 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
1249 edesc
->dst_nents
+ 2];
1251 icvdata
= &edesc
->link_tbl
[0];
1253 sg
= sg_last(req
->src
, edesc
->src_nents
? : 1);
1255 memcpy(icvdata
, (char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
1258 return ipsec_esp(edesc
, req
, NULL
, 0, ipsec_esp_decrypt_swauth_done
);
1261 static int aead_givencrypt(struct aead_givcrypt_request
*req
)
1263 struct aead_request
*areq
= &req
->areq
;
1264 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1265 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1266 struct talitos_edesc
*edesc
;
1268 /* allocate extended descriptor */
1269 edesc
= aead_edesc_alloc(areq
, 0);
1271 return PTR_ERR(edesc
);
1274 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1276 memcpy(req
->giv
, ctx
->iv
, crypto_aead_ivsize(authenc
));
1277 /* avoid consecutive packets going out with same IV */
1278 *(__be64
*)req
->giv
^= cpu_to_be64(req
->seq
);
1280 return ipsec_esp(edesc
, areq
, req
->giv
, req
->seq
,
1281 ipsec_esp_encrypt_done
);
1284 static int ablkcipher_setkey(struct crypto_ablkcipher
*cipher
,
1285 const u8
*key
, unsigned int keylen
)
1287 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1288 struct ablkcipher_alg
*alg
= crypto_ablkcipher_alg(cipher
);
1290 if (keylen
> TALITOS_MAX_KEY_SIZE
)
1293 if (keylen
< alg
->min_keysize
|| keylen
> alg
->max_keysize
)
1296 memcpy(&ctx
->key
, key
, keylen
);
1297 ctx
->keylen
= keylen
;
1302 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
1306 static void common_nonsnoop_unmap(struct device
*dev
,
1307 struct talitos_edesc
*edesc
,
1308 struct ablkcipher_request
*areq
)
1310 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1311 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
1312 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1], DMA_TO_DEVICE
);
1314 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
1317 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1321 static void ablkcipher_done(struct device
*dev
,
1322 struct talitos_desc
*desc
, void *context
,
1325 struct ablkcipher_request
*areq
= context
;
1326 struct talitos_edesc
*edesc
;
1328 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1330 common_nonsnoop_unmap(dev
, edesc
, areq
);
1334 areq
->base
.complete(&areq
->base
, err
);
1337 static int common_nonsnoop(struct talitos_edesc
*edesc
,
1338 struct ablkcipher_request
*areq
,
1340 void (*callback
) (struct device
*dev
,
1341 struct talitos_desc
*desc
,
1342 void *context
, int error
))
1344 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1345 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1346 struct device
*dev
= ctx
->dev
;
1347 struct talitos_desc
*desc
= &edesc
->desc
;
1348 unsigned int cryptlen
= areq
->nbytes
;
1349 unsigned int ivsize
;
1352 /* first DWORD empty */
1353 desc
->ptr
[0].len
= 0;
1354 to_talitos_ptr(&desc
->ptr
[0], 0);
1355 desc
->ptr
[0].j_extent
= 0;
1358 ivsize
= crypto_ablkcipher_ivsize(cipher
);
1359 map_single_talitos_ptr(dev
, &desc
->ptr
[1], ivsize
, giv
?: areq
->info
, 0,
1363 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1364 (char *)&ctx
->key
, 0, DMA_TO_DEVICE
);
1369 desc
->ptr
[3].len
= cpu_to_be16(cryptlen
);
1370 desc
->ptr
[3].j_extent
= 0;
1372 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
? : 1,
1373 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1375 edesc
->src_is_chained
);
1377 if (sg_count
== 1) {
1378 to_talitos_ptr(&desc
->ptr
[3], sg_dma_address(areq
->src
));
1380 sg_count
= sg_to_link_tbl(areq
->src
, sg_count
, cryptlen
,
1381 &edesc
->link_tbl
[0]);
1383 to_talitos_ptr(&desc
->ptr
[3], edesc
->dma_link_tbl
);
1384 desc
->ptr
[3].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1385 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1389 /* Only one segment now, so no link tbl needed */
1390 to_talitos_ptr(&desc
->ptr
[3],
1391 sg_dma_address(areq
->src
));
1396 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1397 desc
->ptr
[4].j_extent
= 0;
1399 if (areq
->src
!= areq
->dst
)
1400 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1401 edesc
->dst_nents
? : 1,
1403 edesc
->dst_is_chained
);
1405 if (sg_count
== 1) {
1406 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->dst
));
1408 struct talitos_ptr
*link_tbl_ptr
=
1409 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1411 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
+
1412 (edesc
->src_nents
+ 1) *
1413 sizeof(struct talitos_ptr
));
1414 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1415 sg_count
= sg_to_link_tbl(areq
->dst
, sg_count
, cryptlen
,
1417 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1418 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1422 map_single_talitos_ptr(dev
, &desc
->ptr
[5], ivsize
, ctx
->iv
, 0,
1425 /* last DWORD empty */
1426 desc
->ptr
[6].len
= 0;
1427 to_talitos_ptr(&desc
->ptr
[6], 0);
1428 desc
->ptr
[6].j_extent
= 0;
1430 ret
= talitos_submit(dev
, desc
, callback
, areq
);
1431 if (ret
!= -EINPROGRESS
) {
1432 common_nonsnoop_unmap(dev
, edesc
, areq
);
1438 static struct talitos_edesc
*ablkcipher_edesc_alloc(struct ablkcipher_request
*
1441 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1442 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1444 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
, areq
->nbytes
,
1445 0, 0, areq
->base
.flags
);
1448 static int ablkcipher_encrypt(struct ablkcipher_request
*areq
)
1450 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1451 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1452 struct talitos_edesc
*edesc
;
1454 /* allocate extended descriptor */
1455 edesc
= ablkcipher_edesc_alloc(areq
);
1457 return PTR_ERR(edesc
);
1460 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1462 return common_nonsnoop(edesc
, areq
, NULL
, ablkcipher_done
);
1465 static int ablkcipher_decrypt(struct ablkcipher_request
*areq
)
1467 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1468 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1469 struct talitos_edesc
*edesc
;
1471 /* allocate extended descriptor */
1472 edesc
= ablkcipher_edesc_alloc(areq
);
1474 return PTR_ERR(edesc
);
1476 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1478 return common_nonsnoop(edesc
, areq
, NULL
, ablkcipher_done
);
1481 struct talitos_alg_template
{
1482 struct crypto_alg alg
;
1483 __be32 desc_hdr_template
;
1486 static struct talitos_alg_template driver_algs
[] = {
1487 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
1490 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
1491 .cra_driver_name
= "authenc-hmac-sha1-cbc-aes-talitos",
1492 .cra_blocksize
= AES_BLOCK_SIZE
,
1493 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1494 .cra_type
= &crypto_aead_type
,
1496 .setkey
= aead_setkey
,
1497 .setauthsize
= aead_setauthsize
,
1498 .encrypt
= aead_encrypt
,
1499 .decrypt
= aead_decrypt
,
1500 .givencrypt
= aead_givencrypt
,
1501 .geniv
= "<built-in>",
1502 .ivsize
= AES_BLOCK_SIZE
,
1503 .maxauthsize
= SHA1_DIGEST_SIZE
,
1506 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1507 DESC_HDR_SEL0_AESU
|
1508 DESC_HDR_MODE0_AESU_CBC
|
1509 DESC_HDR_SEL1_MDEUA
|
1510 DESC_HDR_MODE1_MDEU_INIT
|
1511 DESC_HDR_MODE1_MDEU_PAD
|
1512 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
1516 .cra_name
= "authenc(hmac(sha1),cbc(des3_ede))",
1517 .cra_driver_name
= "authenc-hmac-sha1-cbc-3des-talitos",
1518 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1519 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1520 .cra_type
= &crypto_aead_type
,
1522 .setkey
= aead_setkey
,
1523 .setauthsize
= aead_setauthsize
,
1524 .encrypt
= aead_encrypt
,
1525 .decrypt
= aead_decrypt
,
1526 .givencrypt
= aead_givencrypt
,
1527 .geniv
= "<built-in>",
1528 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1529 .maxauthsize
= SHA1_DIGEST_SIZE
,
1532 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1534 DESC_HDR_MODE0_DEU_CBC
|
1535 DESC_HDR_MODE0_DEU_3DES
|
1536 DESC_HDR_SEL1_MDEUA
|
1537 DESC_HDR_MODE1_MDEU_INIT
|
1538 DESC_HDR_MODE1_MDEU_PAD
|
1539 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
1543 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
1544 .cra_driver_name
= "authenc-hmac-sha256-cbc-aes-talitos",
1545 .cra_blocksize
= AES_BLOCK_SIZE
,
1546 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1547 .cra_type
= &crypto_aead_type
,
1549 .setkey
= aead_setkey
,
1550 .setauthsize
= aead_setauthsize
,
1551 .encrypt
= aead_encrypt
,
1552 .decrypt
= aead_decrypt
,
1553 .givencrypt
= aead_givencrypt
,
1554 .geniv
= "<built-in>",
1555 .ivsize
= AES_BLOCK_SIZE
,
1556 .maxauthsize
= SHA256_DIGEST_SIZE
,
1559 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1560 DESC_HDR_SEL0_AESU
|
1561 DESC_HDR_MODE0_AESU_CBC
|
1562 DESC_HDR_SEL1_MDEUA
|
1563 DESC_HDR_MODE1_MDEU_INIT
|
1564 DESC_HDR_MODE1_MDEU_PAD
|
1565 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
1569 .cra_name
= "authenc(hmac(sha256),cbc(des3_ede))",
1570 .cra_driver_name
= "authenc-hmac-sha256-cbc-3des-talitos",
1571 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1572 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1573 .cra_type
= &crypto_aead_type
,
1575 .setkey
= aead_setkey
,
1576 .setauthsize
= aead_setauthsize
,
1577 .encrypt
= aead_encrypt
,
1578 .decrypt
= aead_decrypt
,
1579 .givencrypt
= aead_givencrypt
,
1580 .geniv
= "<built-in>",
1581 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1582 .maxauthsize
= SHA256_DIGEST_SIZE
,
1585 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1587 DESC_HDR_MODE0_DEU_CBC
|
1588 DESC_HDR_MODE0_DEU_3DES
|
1589 DESC_HDR_SEL1_MDEUA
|
1590 DESC_HDR_MODE1_MDEU_INIT
|
1591 DESC_HDR_MODE1_MDEU_PAD
|
1592 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
1596 .cra_name
= "authenc(hmac(md5),cbc(aes))",
1597 .cra_driver_name
= "authenc-hmac-md5-cbc-aes-talitos",
1598 .cra_blocksize
= AES_BLOCK_SIZE
,
1599 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1600 .cra_type
= &crypto_aead_type
,
1602 .setkey
= aead_setkey
,
1603 .setauthsize
= aead_setauthsize
,
1604 .encrypt
= aead_encrypt
,
1605 .decrypt
= aead_decrypt
,
1606 .givencrypt
= aead_givencrypt
,
1607 .geniv
= "<built-in>",
1608 .ivsize
= AES_BLOCK_SIZE
,
1609 .maxauthsize
= MD5_DIGEST_SIZE
,
1612 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1613 DESC_HDR_SEL0_AESU
|
1614 DESC_HDR_MODE0_AESU_CBC
|
1615 DESC_HDR_SEL1_MDEUA
|
1616 DESC_HDR_MODE1_MDEU_INIT
|
1617 DESC_HDR_MODE1_MDEU_PAD
|
1618 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
1622 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
1623 .cra_driver_name
= "authenc-hmac-md5-cbc-3des-talitos",
1624 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1625 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1626 .cra_type
= &crypto_aead_type
,
1628 .setkey
= aead_setkey
,
1629 .setauthsize
= aead_setauthsize
,
1630 .encrypt
= aead_encrypt
,
1631 .decrypt
= aead_decrypt
,
1632 .givencrypt
= aead_givencrypt
,
1633 .geniv
= "<built-in>",
1634 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1635 .maxauthsize
= MD5_DIGEST_SIZE
,
1638 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1640 DESC_HDR_MODE0_DEU_CBC
|
1641 DESC_HDR_MODE0_DEU_3DES
|
1642 DESC_HDR_SEL1_MDEUA
|
1643 DESC_HDR_MODE1_MDEU_INIT
|
1644 DESC_HDR_MODE1_MDEU_PAD
|
1645 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
1647 /* ABLKCIPHER algorithms. */
1650 .cra_name
= "cbc(aes)",
1651 .cra_driver_name
= "cbc-aes-talitos",
1652 .cra_blocksize
= AES_BLOCK_SIZE
,
1653 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1655 .cra_type
= &crypto_ablkcipher_type
,
1657 .setkey
= ablkcipher_setkey
,
1658 .encrypt
= ablkcipher_encrypt
,
1659 .decrypt
= ablkcipher_decrypt
,
1661 .min_keysize
= AES_MIN_KEY_SIZE
,
1662 .max_keysize
= AES_MAX_KEY_SIZE
,
1663 .ivsize
= AES_BLOCK_SIZE
,
1666 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
1667 DESC_HDR_SEL0_AESU
|
1668 DESC_HDR_MODE0_AESU_CBC
,
1672 .cra_name
= "cbc(des3_ede)",
1673 .cra_driver_name
= "cbc-3des-talitos",
1674 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1675 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1677 .cra_type
= &crypto_ablkcipher_type
,
1679 .setkey
= ablkcipher_setkey
,
1680 .encrypt
= ablkcipher_encrypt
,
1681 .decrypt
= ablkcipher_decrypt
,
1683 .min_keysize
= DES3_EDE_KEY_SIZE
,
1684 .max_keysize
= DES3_EDE_KEY_SIZE
,
1685 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1688 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
1690 DESC_HDR_MODE0_DEU_CBC
|
1691 DESC_HDR_MODE0_DEU_3DES
,
1695 struct talitos_crypto_alg
{
1696 struct list_head entry
;
1698 __be32 desc_hdr_template
;
1699 struct crypto_alg crypto_alg
;
1702 static int talitos_cra_init(struct crypto_tfm
*tfm
)
1704 struct crypto_alg
*alg
= tfm
->__crt_alg
;
1705 struct talitos_crypto_alg
*talitos_alg
;
1706 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1708 talitos_alg
= container_of(alg
, struct talitos_crypto_alg
, crypto_alg
);
1710 /* update context with ptr to dev */
1711 ctx
->dev
= talitos_alg
->dev
;
1713 /* copy descriptor header template value */
1714 ctx
->desc_hdr_template
= talitos_alg
->desc_hdr_template
;
1716 /* random first IV */
1717 get_random_bytes(ctx
->iv
, TALITOS_MAX_IV_LENGTH
);
1723 * given the alg's descriptor header template, determine whether descriptor
1724 * type and primary/secondary execution units required match the hw
1725 * capabilities description provided in the device tree node.
1727 static int hw_supports(struct device
*dev
, __be32 desc_hdr_template
)
1729 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1732 ret
= (1 << DESC_TYPE(desc_hdr_template
) & priv
->desc_types
) &&
1733 (1 << PRIMARY_EU(desc_hdr_template
) & priv
->exec_units
);
1735 if (SECONDARY_EU(desc_hdr_template
))
1736 ret
= ret
&& (1 << SECONDARY_EU(desc_hdr_template
)
1737 & priv
->exec_units
);
1742 static int talitos_remove(struct of_device
*ofdev
)
1744 struct device
*dev
= &ofdev
->dev
;
1745 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1746 struct talitos_crypto_alg
*t_alg
, *n
;
1749 list_for_each_entry_safe(t_alg
, n
, &priv
->alg_list
, entry
) {
1750 crypto_unregister_alg(&t_alg
->crypto_alg
);
1751 list_del(&t_alg
->entry
);
1755 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
))
1756 talitos_unregister_rng(dev
);
1758 for (i
= 0; i
< priv
->num_channels
; i
++)
1759 if (priv
->chan
[i
].fifo
)
1760 kfree(priv
->chan
[i
].fifo
);
1764 if (priv
->irq
!= NO_IRQ
) {
1765 free_irq(priv
->irq
, dev
);
1766 irq_dispose_mapping(priv
->irq
);
1769 tasklet_kill(&priv
->done_task
);
1773 dev_set_drvdata(dev
, NULL
);
1780 static struct talitos_crypto_alg
*talitos_alg_alloc(struct device
*dev
,
1781 struct talitos_alg_template
1784 struct talitos_crypto_alg
*t_alg
;
1785 struct crypto_alg
*alg
;
1787 t_alg
= kzalloc(sizeof(struct talitos_crypto_alg
), GFP_KERNEL
);
1789 return ERR_PTR(-ENOMEM
);
1791 alg
= &t_alg
->crypto_alg
;
1792 *alg
= template->alg
;
1794 alg
->cra_module
= THIS_MODULE
;
1795 alg
->cra_init
= talitos_cra_init
;
1796 alg
->cra_priority
= TALITOS_CRA_PRIORITY
;
1797 alg
->cra_alignmask
= 0;
1798 alg
->cra_ctxsize
= sizeof(struct talitos_ctx
);
1800 t_alg
->desc_hdr_template
= template->desc_hdr_template
;
1806 static int talitos_probe(struct of_device
*ofdev
,
1807 const struct of_device_id
*match
)
1809 struct device
*dev
= &ofdev
->dev
;
1810 struct device_node
*np
= ofdev
->dev
.of_node
;
1811 struct talitos_private
*priv
;
1812 const unsigned int *prop
;
1815 priv
= kzalloc(sizeof(struct talitos_private
), GFP_KERNEL
);
1819 dev_set_drvdata(dev
, priv
);
1821 priv
->ofdev
= ofdev
;
1823 tasklet_init(&priv
->done_task
, talitos_done
, (unsigned long)dev
);
1825 INIT_LIST_HEAD(&priv
->alg_list
);
1827 priv
->irq
= irq_of_parse_and_map(np
, 0);
1829 if (priv
->irq
== NO_IRQ
) {
1830 dev_err(dev
, "failed to map irq\n");
1835 /* get the irq line */
1836 err
= request_irq(priv
->irq
, talitos_interrupt
, 0,
1837 dev_driver_string(dev
), dev
);
1839 dev_err(dev
, "failed to request irq %d\n", priv
->irq
);
1840 irq_dispose_mapping(priv
->irq
);
1845 priv
->reg
= of_iomap(np
, 0);
1847 dev_err(dev
, "failed to of_iomap\n");
1852 /* get SEC version capabilities from device tree */
1853 prop
= of_get_property(np
, "fsl,num-channels", NULL
);
1855 priv
->num_channels
= *prop
;
1857 prop
= of_get_property(np
, "fsl,channel-fifo-len", NULL
);
1859 priv
->chfifo_len
= *prop
;
1861 prop
= of_get_property(np
, "fsl,exec-units-mask", NULL
);
1863 priv
->exec_units
= *prop
;
1865 prop
= of_get_property(np
, "fsl,descriptor-types-mask", NULL
);
1867 priv
->desc_types
= *prop
;
1869 if (!is_power_of_2(priv
->num_channels
) || !priv
->chfifo_len
||
1870 !priv
->exec_units
|| !priv
->desc_types
) {
1871 dev_err(dev
, "invalid property data in device tree node\n");
1876 if (of_device_is_compatible(np
, "fsl,sec3.0"))
1877 priv
->features
|= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
;
1879 if (of_device_is_compatible(np
, "fsl,sec2.1"))
1880 priv
->features
|= TALITOS_FTR_HW_AUTH_CHECK
;
1882 priv
->chan
= kzalloc(sizeof(struct talitos_channel
) *
1883 priv
->num_channels
, GFP_KERNEL
);
1885 dev_err(dev
, "failed to allocate channel management space\n");
1890 for (i
= 0; i
< priv
->num_channels
; i
++) {
1891 spin_lock_init(&priv
->chan
[i
].head_lock
);
1892 spin_lock_init(&priv
->chan
[i
].tail_lock
);
1895 priv
->fifo_len
= roundup_pow_of_two(priv
->chfifo_len
);
1897 for (i
= 0; i
< priv
->num_channels
; i
++) {
1898 priv
->chan
[i
].fifo
= kzalloc(sizeof(struct talitos_request
) *
1899 priv
->fifo_len
, GFP_KERNEL
);
1900 if (!priv
->chan
[i
].fifo
) {
1901 dev_err(dev
, "failed to allocate request fifo %d\n", i
);
1907 for (i
= 0; i
< priv
->num_channels
; i
++)
1908 atomic_set(&priv
->chan
[i
].submit_count
,
1909 -(priv
->chfifo_len
- 1));
1911 dma_set_mask(dev
, DMA_BIT_MASK(36));
1913 /* reset and initialize the h/w */
1914 err
= init_device(dev
);
1916 dev_err(dev
, "failed to initialize device\n");
1920 /* register the RNG, if available */
1921 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
)) {
1922 err
= talitos_register_rng(dev
);
1924 dev_err(dev
, "failed to register hwrng: %d\n", err
);
1927 dev_info(dev
, "hwrng\n");
1930 /* register crypto algorithms the device supports */
1931 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
1932 if (hw_supports(dev
, driver_algs
[i
].desc_hdr_template
)) {
1933 struct talitos_crypto_alg
*t_alg
;
1935 t_alg
= talitos_alg_alloc(dev
, &driver_algs
[i
]);
1936 if (IS_ERR(t_alg
)) {
1937 err
= PTR_ERR(t_alg
);
1941 err
= crypto_register_alg(&t_alg
->crypto_alg
);
1943 dev_err(dev
, "%s alg registration failed\n",
1944 t_alg
->crypto_alg
.cra_driver_name
);
1947 list_add_tail(&t_alg
->entry
, &priv
->alg_list
);
1948 dev_info(dev
, "%s\n",
1949 t_alg
->crypto_alg
.cra_driver_name
);
1957 talitos_remove(ofdev
);
1962 static const struct of_device_id talitos_match
[] = {
1964 .compatible
= "fsl,sec2.0",
1968 MODULE_DEVICE_TABLE(of
, talitos_match
);
1970 static struct of_platform_driver talitos_driver
= {
1973 .owner
= THIS_MODULE
,
1974 .of_match_table
= talitos_match
,
1976 .probe
= talitos_probe
,
1977 .remove
= talitos_remove
,
1980 static int __init
talitos_init(void)
1982 return of_register_platform_driver(&talitos_driver
);
1984 module_init(talitos_init
);
1986 static void __exit
talitos_exit(void)
1988 of_unregister_platform_driver(&talitos_driver
);
1990 module_exit(talitos_exit
);
1992 MODULE_LICENSE("GPL");
1993 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1994 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");