Merge remote-tracking branch 'mailbox/mailbox-for-next'
[deliverable/linux.git] / crypto / cryptd.c
1 /*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
6 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
13 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20 #include <crypto/algapi.h>
21 #include <crypto/internal/hash.h>
22 #include <crypto/internal/aead.h>
23 #include <crypto/cryptd.h>
24 #include <crypto/crypto_wq.h>
25 #include <linux/atomic.h>
26 #include <linux/err.h>
27 #include <linux/init.h>
28 #include <linux/kernel.h>
29 #include <linux/list.h>
30 #include <linux/module.h>
31 #include <linux/scatterlist.h>
32 #include <linux/sched.h>
33 #include <linux/slab.h>
34
35 #define CRYPTD_MAX_CPU_QLEN 1000
36
37 struct cryptd_cpu_queue {
38 struct crypto_queue queue;
39 struct work_struct work;
40 };
41
42 struct cryptd_queue {
43 struct cryptd_cpu_queue __percpu *cpu_queue;
44 };
45
46 struct cryptd_instance_ctx {
47 struct crypto_spawn spawn;
48 struct cryptd_queue *queue;
49 };
50
51 struct hashd_instance_ctx {
52 struct crypto_shash_spawn spawn;
53 struct cryptd_queue *queue;
54 };
55
56 struct aead_instance_ctx {
57 struct crypto_aead_spawn aead_spawn;
58 struct cryptd_queue *queue;
59 };
60
61 struct cryptd_blkcipher_ctx {
62 atomic_t refcnt;
63 struct crypto_blkcipher *child;
64 };
65
66 struct cryptd_blkcipher_request_ctx {
67 crypto_completion_t complete;
68 };
69
70 struct cryptd_hash_ctx {
71 atomic_t refcnt;
72 struct crypto_shash *child;
73 };
74
75 struct cryptd_hash_request_ctx {
76 crypto_completion_t complete;
77 struct shash_desc desc;
78 };
79
80 struct cryptd_aead_ctx {
81 atomic_t refcnt;
82 struct crypto_aead *child;
83 };
84
85 struct cryptd_aead_request_ctx {
86 crypto_completion_t complete;
87 };
88
89 static void cryptd_queue_worker(struct work_struct *work);
90
91 static int cryptd_init_queue(struct cryptd_queue *queue,
92 unsigned int max_cpu_qlen)
93 {
94 int cpu;
95 struct cryptd_cpu_queue *cpu_queue;
96
97 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
98 if (!queue->cpu_queue)
99 return -ENOMEM;
100 for_each_possible_cpu(cpu) {
101 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
102 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
103 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
104 }
105 return 0;
106 }
107
108 static void cryptd_fini_queue(struct cryptd_queue *queue)
109 {
110 int cpu;
111 struct cryptd_cpu_queue *cpu_queue;
112
113 for_each_possible_cpu(cpu) {
114 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
115 BUG_ON(cpu_queue->queue.qlen);
116 }
117 free_percpu(queue->cpu_queue);
118 }
119
120 static int cryptd_enqueue_request(struct cryptd_queue *queue,
121 struct crypto_async_request *request)
122 {
123 int cpu, err;
124 struct cryptd_cpu_queue *cpu_queue;
125 struct crypto_tfm *tfm;
126 atomic_t *refcnt;
127 bool may_backlog;
128
129 cpu = get_cpu();
130 cpu_queue = this_cpu_ptr(queue->cpu_queue);
131 err = crypto_enqueue_request(&cpu_queue->queue, request);
132
133 refcnt = crypto_tfm_ctx(request->tfm);
134 may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
135
136 if (err == -EBUSY && !may_backlog)
137 goto out_put_cpu;
138
139 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
140
141 if (!atomic_read(refcnt))
142 goto out_put_cpu;
143
144 tfm = request->tfm;
145 atomic_inc(refcnt);
146
147 out_put_cpu:
148 put_cpu();
149
150 return err;
151 }
152
153 /* Called in workqueue context, do one real cryption work (via
154 * req->complete) and reschedule itself if there are more work to
155 * do. */
156 static void cryptd_queue_worker(struct work_struct *work)
157 {
158 struct cryptd_cpu_queue *cpu_queue;
159 struct crypto_async_request *req, *backlog;
160
161 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
162 /*
163 * Only handle one request at a time to avoid hogging crypto workqueue.
164 * preempt_disable/enable is used to prevent being preempted by
165 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
166 * cryptd_enqueue_request() being accessed from software interrupts.
167 */
168 local_bh_disable();
169 preempt_disable();
170 backlog = crypto_get_backlog(&cpu_queue->queue);
171 req = crypto_dequeue_request(&cpu_queue->queue);
172 preempt_enable();
173 local_bh_enable();
174
175 if (!req)
176 return;
177
178 if (backlog)
179 backlog->complete(backlog, -EINPROGRESS);
180 req->complete(req, 0);
181
182 if (cpu_queue->queue.qlen)
183 queue_work(kcrypto_wq, &cpu_queue->work);
184 }
185
186 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
187 {
188 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
189 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
190 return ictx->queue;
191 }
192
193 static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
194 u32 *mask)
195 {
196 struct crypto_attr_type *algt;
197
198 algt = crypto_get_attr_type(tb);
199 if (IS_ERR(algt))
200 return;
201
202 *type |= algt->type & CRYPTO_ALG_INTERNAL;
203 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
204 }
205
206 static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
207 const u8 *key, unsigned int keylen)
208 {
209 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
210 struct crypto_blkcipher *child = ctx->child;
211 int err;
212
213 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
214 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
215 CRYPTO_TFM_REQ_MASK);
216 err = crypto_blkcipher_setkey(child, key, keylen);
217 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
218 CRYPTO_TFM_RES_MASK);
219 return err;
220 }
221
222 static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
223 struct crypto_blkcipher *child,
224 int err,
225 int (*crypt)(struct blkcipher_desc *desc,
226 struct scatterlist *dst,
227 struct scatterlist *src,
228 unsigned int len))
229 {
230 struct cryptd_blkcipher_request_ctx *rctx;
231 struct cryptd_blkcipher_ctx *ctx;
232 struct crypto_ablkcipher *tfm;
233 struct blkcipher_desc desc;
234 int refcnt;
235
236 rctx = ablkcipher_request_ctx(req);
237
238 if (unlikely(err == -EINPROGRESS))
239 goto out;
240
241 desc.tfm = child;
242 desc.info = req->info;
243 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
244
245 err = crypt(&desc, req->dst, req->src, req->nbytes);
246
247 req->base.complete = rctx->complete;
248
249 out:
250 tfm = crypto_ablkcipher_reqtfm(req);
251 ctx = crypto_ablkcipher_ctx(tfm);
252 refcnt = atomic_read(&ctx->refcnt);
253
254 local_bh_disable();
255 rctx->complete(&req->base, err);
256 local_bh_enable();
257
258 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
259 crypto_free_ablkcipher(tfm);
260 }
261
262 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
263 {
264 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
265 struct crypto_blkcipher *child = ctx->child;
266
267 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
268 crypto_blkcipher_crt(child)->encrypt);
269 }
270
271 static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
272 {
273 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
274 struct crypto_blkcipher *child = ctx->child;
275
276 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
277 crypto_blkcipher_crt(child)->decrypt);
278 }
279
280 static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
281 crypto_completion_t compl)
282 {
283 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
284 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
285 struct cryptd_queue *queue;
286
287 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
288 rctx->complete = req->base.complete;
289 req->base.complete = compl;
290
291 return cryptd_enqueue_request(queue, &req->base);
292 }
293
294 static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
295 {
296 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
297 }
298
299 static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
300 {
301 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
302 }
303
304 static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
305 {
306 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
307 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
308 struct crypto_spawn *spawn = &ictx->spawn;
309 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
310 struct crypto_blkcipher *cipher;
311
312 cipher = crypto_spawn_blkcipher(spawn);
313 if (IS_ERR(cipher))
314 return PTR_ERR(cipher);
315
316 ctx->child = cipher;
317 tfm->crt_ablkcipher.reqsize =
318 sizeof(struct cryptd_blkcipher_request_ctx);
319 return 0;
320 }
321
322 static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
323 {
324 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
325
326 crypto_free_blkcipher(ctx->child);
327 }
328
329 static int cryptd_init_instance(struct crypto_instance *inst,
330 struct crypto_alg *alg)
331 {
332 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
333 "cryptd(%s)",
334 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
335 return -ENAMETOOLONG;
336
337 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
338
339 inst->alg.cra_priority = alg->cra_priority + 50;
340 inst->alg.cra_blocksize = alg->cra_blocksize;
341 inst->alg.cra_alignmask = alg->cra_alignmask;
342
343 return 0;
344 }
345
346 static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
347 unsigned int tail)
348 {
349 char *p;
350 struct crypto_instance *inst;
351 int err;
352
353 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
354 if (!p)
355 return ERR_PTR(-ENOMEM);
356
357 inst = (void *)(p + head);
358
359 err = cryptd_init_instance(inst, alg);
360 if (err)
361 goto out_free_inst;
362
363 out:
364 return p;
365
366 out_free_inst:
367 kfree(p);
368 p = ERR_PTR(err);
369 goto out;
370 }
371
372 static int cryptd_create_blkcipher(struct crypto_template *tmpl,
373 struct rtattr **tb,
374 struct cryptd_queue *queue)
375 {
376 struct cryptd_instance_ctx *ctx;
377 struct crypto_instance *inst;
378 struct crypto_alg *alg;
379 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
380 u32 mask = CRYPTO_ALG_TYPE_MASK;
381 int err;
382
383 cryptd_check_internal(tb, &type, &mask);
384
385 alg = crypto_get_attr_alg(tb, type, mask);
386 if (IS_ERR(alg))
387 return PTR_ERR(alg);
388
389 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
390 err = PTR_ERR(inst);
391 if (IS_ERR(inst))
392 goto out_put_alg;
393
394 ctx = crypto_instance_ctx(inst);
395 ctx->queue = queue;
396
397 err = crypto_init_spawn(&ctx->spawn, alg, inst,
398 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
399 if (err)
400 goto out_free_inst;
401
402 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
403 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
404 type |= CRYPTO_ALG_INTERNAL;
405 inst->alg.cra_flags = type;
406 inst->alg.cra_type = &crypto_ablkcipher_type;
407
408 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
409 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
410 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
411
412 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
413
414 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
415
416 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
417 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
418
419 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
420 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
421 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
422
423 err = crypto_register_instance(tmpl, inst);
424 if (err) {
425 crypto_drop_spawn(&ctx->spawn);
426 out_free_inst:
427 kfree(inst);
428 }
429
430 out_put_alg:
431 crypto_mod_put(alg);
432 return err;
433 }
434
435 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
436 {
437 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
438 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
439 struct crypto_shash_spawn *spawn = &ictx->spawn;
440 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
441 struct crypto_shash *hash;
442
443 hash = crypto_spawn_shash(spawn);
444 if (IS_ERR(hash))
445 return PTR_ERR(hash);
446
447 ctx->child = hash;
448 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
449 sizeof(struct cryptd_hash_request_ctx) +
450 crypto_shash_descsize(hash));
451 return 0;
452 }
453
454 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
455 {
456 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
457
458 crypto_free_shash(ctx->child);
459 }
460
461 static int cryptd_hash_setkey(struct crypto_ahash *parent,
462 const u8 *key, unsigned int keylen)
463 {
464 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
465 struct crypto_shash *child = ctx->child;
466 int err;
467
468 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
469 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
470 CRYPTO_TFM_REQ_MASK);
471 err = crypto_shash_setkey(child, key, keylen);
472 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
473 CRYPTO_TFM_RES_MASK);
474 return err;
475 }
476
477 static int cryptd_hash_enqueue(struct ahash_request *req,
478 crypto_completion_t compl)
479 {
480 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
481 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
482 struct cryptd_queue *queue =
483 cryptd_get_queue(crypto_ahash_tfm(tfm));
484
485 rctx->complete = req->base.complete;
486 req->base.complete = compl;
487
488 return cryptd_enqueue_request(queue, &req->base);
489 }
490
491 static void cryptd_hash_complete(struct ahash_request *req, int err)
492 {
493 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
494 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
495 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
496 int refcnt = atomic_read(&ctx->refcnt);
497
498 local_bh_disable();
499 rctx->complete(&req->base, err);
500 local_bh_enable();
501
502 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
503 crypto_free_ahash(tfm);
504 }
505
506 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
507 {
508 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
509 struct crypto_shash *child = ctx->child;
510 struct ahash_request *req = ahash_request_cast(req_async);
511 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
512 struct shash_desc *desc = &rctx->desc;
513
514 if (unlikely(err == -EINPROGRESS))
515 goto out;
516
517 desc->tfm = child;
518 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
519
520 err = crypto_shash_init(desc);
521
522 req->base.complete = rctx->complete;
523
524 out:
525 cryptd_hash_complete(req, err);
526 }
527
528 static int cryptd_hash_init_enqueue(struct ahash_request *req)
529 {
530 return cryptd_hash_enqueue(req, cryptd_hash_init);
531 }
532
533 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
534 {
535 struct ahash_request *req = ahash_request_cast(req_async);
536 struct cryptd_hash_request_ctx *rctx;
537
538 rctx = ahash_request_ctx(req);
539
540 if (unlikely(err == -EINPROGRESS))
541 goto out;
542
543 err = shash_ahash_update(req, &rctx->desc);
544
545 req->base.complete = rctx->complete;
546
547 out:
548 cryptd_hash_complete(req, err);
549 }
550
551 static int cryptd_hash_update_enqueue(struct ahash_request *req)
552 {
553 return cryptd_hash_enqueue(req, cryptd_hash_update);
554 }
555
556 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
557 {
558 struct ahash_request *req = ahash_request_cast(req_async);
559 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
560
561 if (unlikely(err == -EINPROGRESS))
562 goto out;
563
564 err = crypto_shash_final(&rctx->desc, req->result);
565
566 req->base.complete = rctx->complete;
567
568 out:
569 cryptd_hash_complete(req, err);
570 }
571
572 static int cryptd_hash_final_enqueue(struct ahash_request *req)
573 {
574 return cryptd_hash_enqueue(req, cryptd_hash_final);
575 }
576
577 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
578 {
579 struct ahash_request *req = ahash_request_cast(req_async);
580 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
581
582 if (unlikely(err == -EINPROGRESS))
583 goto out;
584
585 err = shash_ahash_finup(req, &rctx->desc);
586
587 req->base.complete = rctx->complete;
588
589 out:
590 cryptd_hash_complete(req, err);
591 }
592
593 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
594 {
595 return cryptd_hash_enqueue(req, cryptd_hash_finup);
596 }
597
598 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
599 {
600 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
601 struct crypto_shash *child = ctx->child;
602 struct ahash_request *req = ahash_request_cast(req_async);
603 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
604 struct shash_desc *desc = &rctx->desc;
605
606 if (unlikely(err == -EINPROGRESS))
607 goto out;
608
609 desc->tfm = child;
610 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
611
612 err = shash_ahash_digest(req, desc);
613
614 req->base.complete = rctx->complete;
615
616 out:
617 cryptd_hash_complete(req, err);
618 }
619
620 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
621 {
622 return cryptd_hash_enqueue(req, cryptd_hash_digest);
623 }
624
625 static int cryptd_hash_export(struct ahash_request *req, void *out)
626 {
627 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
628
629 return crypto_shash_export(&rctx->desc, out);
630 }
631
632 static int cryptd_hash_import(struct ahash_request *req, const void *in)
633 {
634 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
635 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
636 struct shash_desc *desc = cryptd_shash_desc(req);
637
638 desc->tfm = ctx->child;
639 desc->flags = req->base.flags;
640
641 return crypto_shash_import(desc, in);
642 }
643
644 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
645 struct cryptd_queue *queue)
646 {
647 struct hashd_instance_ctx *ctx;
648 struct ahash_instance *inst;
649 struct shash_alg *salg;
650 struct crypto_alg *alg;
651 u32 type = 0;
652 u32 mask = 0;
653 int err;
654
655 cryptd_check_internal(tb, &type, &mask);
656
657 salg = shash_attr_alg(tb[1], type, mask);
658 if (IS_ERR(salg))
659 return PTR_ERR(salg);
660
661 alg = &salg->base;
662 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
663 sizeof(*ctx));
664 err = PTR_ERR(inst);
665 if (IS_ERR(inst))
666 goto out_put_alg;
667
668 ctx = ahash_instance_ctx(inst);
669 ctx->queue = queue;
670
671 err = crypto_init_shash_spawn(&ctx->spawn, salg,
672 ahash_crypto_instance(inst));
673 if (err)
674 goto out_free_inst;
675
676 type = CRYPTO_ALG_ASYNC;
677 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
678 type |= CRYPTO_ALG_INTERNAL;
679 inst->alg.halg.base.cra_flags = type;
680
681 inst->alg.halg.digestsize = salg->digestsize;
682 inst->alg.halg.statesize = salg->statesize;
683 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
684
685 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
686 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
687
688 inst->alg.init = cryptd_hash_init_enqueue;
689 inst->alg.update = cryptd_hash_update_enqueue;
690 inst->alg.final = cryptd_hash_final_enqueue;
691 inst->alg.finup = cryptd_hash_finup_enqueue;
692 inst->alg.export = cryptd_hash_export;
693 inst->alg.import = cryptd_hash_import;
694 inst->alg.setkey = cryptd_hash_setkey;
695 inst->alg.digest = cryptd_hash_digest_enqueue;
696
697 err = ahash_register_instance(tmpl, inst);
698 if (err) {
699 crypto_drop_shash(&ctx->spawn);
700 out_free_inst:
701 kfree(inst);
702 }
703
704 out_put_alg:
705 crypto_mod_put(alg);
706 return err;
707 }
708
709 static int cryptd_aead_setkey(struct crypto_aead *parent,
710 const u8 *key, unsigned int keylen)
711 {
712 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
713 struct crypto_aead *child = ctx->child;
714
715 return crypto_aead_setkey(child, key, keylen);
716 }
717
718 static int cryptd_aead_setauthsize(struct crypto_aead *parent,
719 unsigned int authsize)
720 {
721 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
722 struct crypto_aead *child = ctx->child;
723
724 return crypto_aead_setauthsize(child, authsize);
725 }
726
727 static void cryptd_aead_crypt(struct aead_request *req,
728 struct crypto_aead *child,
729 int err,
730 int (*crypt)(struct aead_request *req))
731 {
732 struct cryptd_aead_request_ctx *rctx;
733 struct cryptd_aead_ctx *ctx;
734 crypto_completion_t compl;
735 struct crypto_aead *tfm;
736 int refcnt;
737
738 rctx = aead_request_ctx(req);
739 compl = rctx->complete;
740
741 tfm = crypto_aead_reqtfm(req);
742
743 if (unlikely(err == -EINPROGRESS))
744 goto out;
745 aead_request_set_tfm(req, child);
746 err = crypt( req );
747
748 out:
749 ctx = crypto_aead_ctx(tfm);
750 refcnt = atomic_read(&ctx->refcnt);
751
752 local_bh_disable();
753 compl(&req->base, err);
754 local_bh_enable();
755
756 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
757 crypto_free_aead(tfm);
758 }
759
760 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
761 {
762 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
763 struct crypto_aead *child = ctx->child;
764 struct aead_request *req;
765
766 req = container_of(areq, struct aead_request, base);
767 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
768 }
769
770 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
771 {
772 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
773 struct crypto_aead *child = ctx->child;
774 struct aead_request *req;
775
776 req = container_of(areq, struct aead_request, base);
777 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
778 }
779
780 static int cryptd_aead_enqueue(struct aead_request *req,
781 crypto_completion_t compl)
782 {
783 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
784 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
785 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
786
787 rctx->complete = req->base.complete;
788 req->base.complete = compl;
789 return cryptd_enqueue_request(queue, &req->base);
790 }
791
792 static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
793 {
794 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
795 }
796
797 static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
798 {
799 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
800 }
801
802 static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
803 {
804 struct aead_instance *inst = aead_alg_instance(tfm);
805 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
806 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
807 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
808 struct crypto_aead *cipher;
809
810 cipher = crypto_spawn_aead(spawn);
811 if (IS_ERR(cipher))
812 return PTR_ERR(cipher);
813
814 ctx->child = cipher;
815 crypto_aead_set_reqsize(
816 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
817 crypto_aead_reqsize(cipher)));
818 return 0;
819 }
820
821 static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
822 {
823 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
824 crypto_free_aead(ctx->child);
825 }
826
827 static int cryptd_create_aead(struct crypto_template *tmpl,
828 struct rtattr **tb,
829 struct cryptd_queue *queue)
830 {
831 struct aead_instance_ctx *ctx;
832 struct aead_instance *inst;
833 struct aead_alg *alg;
834 const char *name;
835 u32 type = 0;
836 u32 mask = CRYPTO_ALG_ASYNC;
837 int err;
838
839 cryptd_check_internal(tb, &type, &mask);
840
841 name = crypto_attr_alg_name(tb[1]);
842 if (IS_ERR(name))
843 return PTR_ERR(name);
844
845 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
846 if (!inst)
847 return -ENOMEM;
848
849 ctx = aead_instance_ctx(inst);
850 ctx->queue = queue;
851
852 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
853 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
854 if (err)
855 goto out_free_inst;
856
857 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
858 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
859 if (err)
860 goto out_drop_aead;
861
862 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
863 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
864 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
865
866 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
867 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
868
869 inst->alg.init = cryptd_aead_init_tfm;
870 inst->alg.exit = cryptd_aead_exit_tfm;
871 inst->alg.setkey = cryptd_aead_setkey;
872 inst->alg.setauthsize = cryptd_aead_setauthsize;
873 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
874 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
875
876 err = aead_register_instance(tmpl, inst);
877 if (err) {
878 out_drop_aead:
879 crypto_drop_aead(&ctx->aead_spawn);
880 out_free_inst:
881 kfree(inst);
882 }
883 return err;
884 }
885
886 static struct cryptd_queue queue;
887
888 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
889 {
890 struct crypto_attr_type *algt;
891
892 algt = crypto_get_attr_type(tb);
893 if (IS_ERR(algt))
894 return PTR_ERR(algt);
895
896 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
897 case CRYPTO_ALG_TYPE_BLKCIPHER:
898 return cryptd_create_blkcipher(tmpl, tb, &queue);
899 case CRYPTO_ALG_TYPE_DIGEST:
900 return cryptd_create_hash(tmpl, tb, &queue);
901 case CRYPTO_ALG_TYPE_AEAD:
902 return cryptd_create_aead(tmpl, tb, &queue);
903 }
904
905 return -EINVAL;
906 }
907
908 static void cryptd_free(struct crypto_instance *inst)
909 {
910 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
911 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
912 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
913
914 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
915 case CRYPTO_ALG_TYPE_AHASH:
916 crypto_drop_shash(&hctx->spawn);
917 kfree(ahash_instance(inst));
918 return;
919 case CRYPTO_ALG_TYPE_AEAD:
920 crypto_drop_aead(&aead_ctx->aead_spawn);
921 kfree(aead_instance(inst));
922 return;
923 default:
924 crypto_drop_spawn(&ctx->spawn);
925 kfree(inst);
926 }
927 }
928
929 static struct crypto_template cryptd_tmpl = {
930 .name = "cryptd",
931 .create = cryptd_create,
932 .free = cryptd_free,
933 .module = THIS_MODULE,
934 };
935
936 struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
937 u32 type, u32 mask)
938 {
939 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
940 struct cryptd_blkcipher_ctx *ctx;
941 struct crypto_tfm *tfm;
942
943 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
944 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
945 return ERR_PTR(-EINVAL);
946 type = crypto_skcipher_type(type);
947 mask &= ~CRYPTO_ALG_TYPE_MASK;
948 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
949 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
950 if (IS_ERR(tfm))
951 return ERR_CAST(tfm);
952 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
953 crypto_free_tfm(tfm);
954 return ERR_PTR(-EINVAL);
955 }
956
957 ctx = crypto_tfm_ctx(tfm);
958 atomic_set(&ctx->refcnt, 1);
959
960 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
961 }
962 EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
963
964 struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
965 {
966 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
967 return ctx->child;
968 }
969 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
970
971 bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
972 {
973 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
974
975 return atomic_read(&ctx->refcnt) - 1;
976 }
977 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
978
979 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
980 {
981 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
982
983 if (atomic_dec_and_test(&ctx->refcnt))
984 crypto_free_ablkcipher(&tfm->base);
985 }
986 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
987
988 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
989 u32 type, u32 mask)
990 {
991 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
992 struct cryptd_hash_ctx *ctx;
993 struct crypto_ahash *tfm;
994
995 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
996 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
997 return ERR_PTR(-EINVAL);
998 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
999 if (IS_ERR(tfm))
1000 return ERR_CAST(tfm);
1001 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1002 crypto_free_ahash(tfm);
1003 return ERR_PTR(-EINVAL);
1004 }
1005
1006 ctx = crypto_ahash_ctx(tfm);
1007 atomic_set(&ctx->refcnt, 1);
1008
1009 return __cryptd_ahash_cast(tfm);
1010 }
1011 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1012
1013 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1014 {
1015 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1016
1017 return ctx->child;
1018 }
1019 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1020
1021 struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1022 {
1023 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1024 return &rctx->desc;
1025 }
1026 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1027
1028 bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1029 {
1030 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1031
1032 return atomic_read(&ctx->refcnt) - 1;
1033 }
1034 EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1035
1036 void cryptd_free_ahash(struct cryptd_ahash *tfm)
1037 {
1038 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1039
1040 if (atomic_dec_and_test(&ctx->refcnt))
1041 crypto_free_ahash(&tfm->base);
1042 }
1043 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1044
1045 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1046 u32 type, u32 mask)
1047 {
1048 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1049 struct cryptd_aead_ctx *ctx;
1050 struct crypto_aead *tfm;
1051
1052 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1053 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1054 return ERR_PTR(-EINVAL);
1055 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1056 if (IS_ERR(tfm))
1057 return ERR_CAST(tfm);
1058 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1059 crypto_free_aead(tfm);
1060 return ERR_PTR(-EINVAL);
1061 }
1062
1063 ctx = crypto_aead_ctx(tfm);
1064 atomic_set(&ctx->refcnt, 1);
1065
1066 return __cryptd_aead_cast(tfm);
1067 }
1068 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1069
1070 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1071 {
1072 struct cryptd_aead_ctx *ctx;
1073 ctx = crypto_aead_ctx(&tfm->base);
1074 return ctx->child;
1075 }
1076 EXPORT_SYMBOL_GPL(cryptd_aead_child);
1077
1078 bool cryptd_aead_queued(struct cryptd_aead *tfm)
1079 {
1080 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1081
1082 return atomic_read(&ctx->refcnt) - 1;
1083 }
1084 EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1085
1086 void cryptd_free_aead(struct cryptd_aead *tfm)
1087 {
1088 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1089
1090 if (atomic_dec_and_test(&ctx->refcnt))
1091 crypto_free_aead(&tfm->base);
1092 }
1093 EXPORT_SYMBOL_GPL(cryptd_free_aead);
1094
1095 static int __init cryptd_init(void)
1096 {
1097 int err;
1098
1099 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
1100 if (err)
1101 return err;
1102
1103 err = crypto_register_template(&cryptd_tmpl);
1104 if (err)
1105 cryptd_fini_queue(&queue);
1106
1107 return err;
1108 }
1109
1110 static void __exit cryptd_exit(void)
1111 {
1112 cryptd_fini_queue(&queue);
1113 crypto_unregister_template(&cryptd_tmpl);
1114 }
1115
1116 subsys_initcall(cryptd_init);
1117 module_exit(cryptd_exit);
1118
1119 MODULE_LICENSE("GPL");
1120 MODULE_DESCRIPTION("Software async crypto daemon");
1121 MODULE_ALIAS_CRYPTO("cryptd");
This page took 0.05418 seconds and 5 git commands to generate.