2 * Copyright (C) 2003 Jana Saout <jana@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
5 * Copyright (C) 2013 Milan Broz <gmazyland@gmail.com>
7 * This file is released under the GPL.
10 #include <linux/completion.h>
11 #include <linux/err.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/bio.h>
16 #include <linux/blkdev.h>
17 #include <linux/mempool.h>
18 #include <linux/slab.h>
19 #include <linux/crypto.h>
20 #include <linux/workqueue.h>
21 #include <linux/backing-dev.h>
22 #include <linux/atomic.h>
23 #include <linux/scatterlist.h>
25 #include <asm/unaligned.h>
26 #include <crypto/hash.h>
27 #include <crypto/md5.h>
28 #include <crypto/algapi.h>
30 #include <linux/device-mapper.h>
32 #define DM_MSG_PREFIX "crypt"
35 * context holding the current state of a multi-part conversion
37 struct convert_context
{
38 struct completion restart
;
41 struct bvec_iter iter_in
;
42 struct bvec_iter iter_out
;
45 struct ablkcipher_request
*req
;
49 * per bio private data
52 struct crypt_config
*cc
;
54 struct work_struct work
;
56 struct convert_context ctx
;
61 struct dm_crypt_io
*base_io
;
62 } CRYPTO_MINALIGN_ATTR
;
64 struct dm_crypt_request
{
65 struct convert_context
*ctx
;
66 struct scatterlist sg_in
;
67 struct scatterlist sg_out
;
73 struct crypt_iv_operations
{
74 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
76 void (*dtr
)(struct crypt_config
*cc
);
77 int (*init
)(struct crypt_config
*cc
);
78 int (*wipe
)(struct crypt_config
*cc
);
79 int (*generator
)(struct crypt_config
*cc
, u8
*iv
,
80 struct dm_crypt_request
*dmreq
);
81 int (*post
)(struct crypt_config
*cc
, u8
*iv
,
82 struct dm_crypt_request
*dmreq
);
85 struct iv_essiv_private
{
86 struct crypto_hash
*hash_tfm
;
90 struct iv_benbi_private
{
94 #define LMK_SEED_SIZE 64 /* hash + 0 */
95 struct iv_lmk_private
{
96 struct crypto_shash
*hash_tfm
;
100 #define TCW_WHITENING_SIZE 16
101 struct iv_tcw_private
{
102 struct crypto_shash
*crc32_tfm
;
108 * Crypt: maps a linear range of a block device
109 * and encrypts / decrypts at the same time.
111 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
};
114 * The fields in here must be read only after initialization.
116 struct crypt_config
{
121 * pool for per bio private data, crypto requests and
122 * encryption requeusts/buffer pages
126 mempool_t
*page_pool
;
129 struct workqueue_struct
*io_queue
;
130 struct workqueue_struct
*crypt_queue
;
135 struct crypt_iv_operations
*iv_gen_ops
;
137 struct iv_essiv_private essiv
;
138 struct iv_benbi_private benbi
;
139 struct iv_lmk_private lmk
;
140 struct iv_tcw_private tcw
;
143 unsigned int iv_size
;
145 /* ESSIV: struct crypto_cipher *essiv_tfm */
147 struct crypto_ablkcipher
**tfms
;
151 * Layout of each crypto request:
153 * struct ablkcipher_request
156 * struct dm_crypt_request
160 * The padding is added so that dm_crypt_request and the IV are
163 unsigned int dmreq_start
;
165 unsigned int per_bio_data_size
;
168 unsigned int key_size
;
169 unsigned int key_parts
; /* independent parts in key buffer */
170 unsigned int key_extra_size
; /* additional keys length */
175 #define MIN_POOL_PAGES 32
177 static struct kmem_cache
*_crypt_io_pool
;
179 static void clone_init(struct dm_crypt_io
*, struct bio
*);
180 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
181 static u8
*iv_of_dmreq(struct crypt_config
*cc
, struct dm_crypt_request
*dmreq
);
184 * Use this to access cipher attributes that are the same for each CPU.
186 static struct crypto_ablkcipher
*any_tfm(struct crypt_config
*cc
)
192 * Different IV generation algorithms:
194 * plain: the initial vector is the 32-bit little-endian version of the sector
195 * number, padded with zeros if necessary.
197 * plain64: the initial vector is the 64-bit little-endian version of the sector
198 * number, padded with zeros if necessary.
200 * essiv: "encrypted sector|salt initial vector", the sector number is
201 * encrypted with the bulk cipher using a salt as key. The salt
202 * should be derived from the bulk cipher's key via hashing.
204 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
205 * (needed for LRW-32-AES and possible other narrow block modes)
207 * null: the initial vector is always zero. Provides compatibility with
208 * obsolete loop_fish2 devices. Do not use for new devices.
210 * lmk: Compatible implementation of the block chaining mode used
211 * by the Loop-AES block device encryption system
212 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
213 * It operates on full 512 byte sectors and uses CBC
214 * with an IV derived from the sector number, the data and
215 * optionally extra IV seed.
216 * This means that after decryption the first block
217 * of sector must be tweaked according to decrypted data.
218 * Loop-AES can use three encryption schemes:
219 * version 1: is plain aes-cbc mode
220 * version 2: uses 64 multikey scheme with lmk IV generator
221 * version 3: the same as version 2 with additional IV seed
222 * (it uses 65 keys, last key is used as IV seed)
224 * tcw: Compatible implementation of the block chaining mode used
225 * by the TrueCrypt device encryption system (prior to version 4.1).
226 * For more info see: http://www.truecrypt.org
227 * It operates on full 512 byte sectors and uses CBC
228 * with an IV derived from initial key and the sector number.
229 * In addition, whitening value is applied on every sector, whitening
230 * is calculated from initial key, sector number and mixed using CRC32.
231 * Note that this encryption scheme is vulnerable to watermarking attacks
232 * and should be used for old compatible containers access only.
234 * plumb: unimplemented, see:
235 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
238 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
,
239 struct dm_crypt_request
*dmreq
)
241 memset(iv
, 0, cc
->iv_size
);
242 *(__le32
*)iv
= cpu_to_le32(dmreq
->iv_sector
& 0xffffffff);
247 static int crypt_iv_plain64_gen(struct crypt_config
*cc
, u8
*iv
,
248 struct dm_crypt_request
*dmreq
)
250 memset(iv
, 0, cc
->iv_size
);
251 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
256 /* Initialise ESSIV - compute salt but no local memory allocations */
257 static int crypt_iv_essiv_init(struct crypt_config
*cc
)
259 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
260 struct hash_desc desc
;
261 struct scatterlist sg
;
262 struct crypto_cipher
*essiv_tfm
;
265 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
266 desc
.tfm
= essiv
->hash_tfm
;
267 desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
269 err
= crypto_hash_digest(&desc
, &sg
, cc
->key_size
, essiv
->salt
);
273 essiv_tfm
= cc
->iv_private
;
275 err
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
,
276 crypto_hash_digestsize(essiv
->hash_tfm
));
283 /* Wipe salt and reset key derived from volume key */
284 static int crypt_iv_essiv_wipe(struct crypt_config
*cc
)
286 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
287 unsigned salt_size
= crypto_hash_digestsize(essiv
->hash_tfm
);
288 struct crypto_cipher
*essiv_tfm
;
291 memset(essiv
->salt
, 0, salt_size
);
293 essiv_tfm
= cc
->iv_private
;
294 r
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
, salt_size
);
301 /* Set up per cpu cipher state */
302 static struct crypto_cipher
*setup_essiv_cpu(struct crypt_config
*cc
,
303 struct dm_target
*ti
,
304 u8
*salt
, unsigned saltsize
)
306 struct crypto_cipher
*essiv_tfm
;
309 /* Setup the essiv_tfm with the given salt */
310 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
311 if (IS_ERR(essiv_tfm
)) {
312 ti
->error
= "Error allocating crypto tfm for ESSIV";
316 if (crypto_cipher_blocksize(essiv_tfm
) !=
317 crypto_ablkcipher_ivsize(any_tfm(cc
))) {
318 ti
->error
= "Block size of ESSIV cipher does "
319 "not match IV size of block cipher";
320 crypto_free_cipher(essiv_tfm
);
321 return ERR_PTR(-EINVAL
);
324 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
326 ti
->error
= "Failed to set key for ESSIV cipher";
327 crypto_free_cipher(essiv_tfm
);
334 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
336 struct crypto_cipher
*essiv_tfm
;
337 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
339 crypto_free_hash(essiv
->hash_tfm
);
340 essiv
->hash_tfm
= NULL
;
345 essiv_tfm
= cc
->iv_private
;
348 crypto_free_cipher(essiv_tfm
);
350 cc
->iv_private
= NULL
;
353 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
356 struct crypto_cipher
*essiv_tfm
= NULL
;
357 struct crypto_hash
*hash_tfm
= NULL
;
362 ti
->error
= "Digest algorithm missing for ESSIV mode";
366 /* Allocate hash algorithm */
367 hash_tfm
= crypto_alloc_hash(opts
, 0, CRYPTO_ALG_ASYNC
);
368 if (IS_ERR(hash_tfm
)) {
369 ti
->error
= "Error initializing ESSIV hash";
370 err
= PTR_ERR(hash_tfm
);
374 salt
= kzalloc(crypto_hash_digestsize(hash_tfm
), GFP_KERNEL
);
376 ti
->error
= "Error kmallocing salt storage in ESSIV";
381 cc
->iv_gen_private
.essiv
.salt
= salt
;
382 cc
->iv_gen_private
.essiv
.hash_tfm
= hash_tfm
;
384 essiv_tfm
= setup_essiv_cpu(cc
, ti
, salt
,
385 crypto_hash_digestsize(hash_tfm
));
386 if (IS_ERR(essiv_tfm
)) {
387 crypt_iv_essiv_dtr(cc
);
388 return PTR_ERR(essiv_tfm
);
390 cc
->iv_private
= essiv_tfm
;
395 if (hash_tfm
&& !IS_ERR(hash_tfm
))
396 crypto_free_hash(hash_tfm
);
401 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
,
402 struct dm_crypt_request
*dmreq
)
404 struct crypto_cipher
*essiv_tfm
= cc
->iv_private
;
406 memset(iv
, 0, cc
->iv_size
);
407 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
408 crypto_cipher_encrypt_one(essiv_tfm
, iv
, iv
);
413 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
416 unsigned bs
= crypto_ablkcipher_blocksize(any_tfm(cc
));
419 /* we need to calculate how far we must shift the sector count
420 * to get the cipher block count, we use this shift in _gen */
422 if (1 << log
!= bs
) {
423 ti
->error
= "cypher blocksize is not a power of 2";
428 ti
->error
= "cypher blocksize is > 512";
432 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
437 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
441 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
,
442 struct dm_crypt_request
*dmreq
)
446 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
448 val
= cpu_to_be64(((u64
)dmreq
->iv_sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
449 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
454 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
,
455 struct dm_crypt_request
*dmreq
)
457 memset(iv
, 0, cc
->iv_size
);
462 static void crypt_iv_lmk_dtr(struct crypt_config
*cc
)
464 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
466 if (lmk
->hash_tfm
&& !IS_ERR(lmk
->hash_tfm
))
467 crypto_free_shash(lmk
->hash_tfm
);
468 lmk
->hash_tfm
= NULL
;
474 static int crypt_iv_lmk_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
477 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
479 lmk
->hash_tfm
= crypto_alloc_shash("md5", 0, 0);
480 if (IS_ERR(lmk
->hash_tfm
)) {
481 ti
->error
= "Error initializing LMK hash";
482 return PTR_ERR(lmk
->hash_tfm
);
485 /* No seed in LMK version 2 */
486 if (cc
->key_parts
== cc
->tfms_count
) {
491 lmk
->seed
= kzalloc(LMK_SEED_SIZE
, GFP_KERNEL
);
493 crypt_iv_lmk_dtr(cc
);
494 ti
->error
= "Error kmallocing seed storage in LMK";
501 static int crypt_iv_lmk_init(struct crypt_config
*cc
)
503 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
504 int subkey_size
= cc
->key_size
/ cc
->key_parts
;
506 /* LMK seed is on the position of LMK_KEYS + 1 key */
508 memcpy(lmk
->seed
, cc
->key
+ (cc
->tfms_count
* subkey_size
),
509 crypto_shash_digestsize(lmk
->hash_tfm
));
514 static int crypt_iv_lmk_wipe(struct crypt_config
*cc
)
516 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
519 memset(lmk
->seed
, 0, LMK_SEED_SIZE
);
524 static int crypt_iv_lmk_one(struct crypt_config
*cc
, u8
*iv
,
525 struct dm_crypt_request
*dmreq
,
528 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
529 SHASH_DESC_ON_STACK(desc
, lmk
->hash_tfm
);
530 struct md5_state md5state
;
534 desc
->tfm
= lmk
->hash_tfm
;
535 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
537 r
= crypto_shash_init(desc
);
542 r
= crypto_shash_update(desc
, lmk
->seed
, LMK_SEED_SIZE
);
547 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
548 r
= crypto_shash_update(desc
, data
+ 16, 16 * 31);
552 /* Sector is cropped to 56 bits here */
553 buf
[0] = cpu_to_le32(dmreq
->iv_sector
& 0xFFFFFFFF);
554 buf
[1] = cpu_to_le32((((u64
)dmreq
->iv_sector
>> 32) & 0x00FFFFFF) | 0x80000000);
555 buf
[2] = cpu_to_le32(4024);
557 r
= crypto_shash_update(desc
, (u8
*)buf
, sizeof(buf
));
561 /* No MD5 padding here */
562 r
= crypto_shash_export(desc
, &md5state
);
566 for (i
= 0; i
< MD5_HASH_WORDS
; i
++)
567 __cpu_to_le32s(&md5state
.hash
[i
]);
568 memcpy(iv
, &md5state
.hash
, cc
->iv_size
);
573 static int crypt_iv_lmk_gen(struct crypt_config
*cc
, u8
*iv
,
574 struct dm_crypt_request
*dmreq
)
579 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
) {
580 src
= kmap_atomic(sg_page(&dmreq
->sg_in
));
581 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, src
+ dmreq
->sg_in
.offset
);
584 memset(iv
, 0, cc
->iv_size
);
589 static int crypt_iv_lmk_post(struct crypt_config
*cc
, u8
*iv
,
590 struct dm_crypt_request
*dmreq
)
595 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
)
598 dst
= kmap_atomic(sg_page(&dmreq
->sg_out
));
599 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, dst
+ dmreq
->sg_out
.offset
);
601 /* Tweak the first block of plaintext sector */
603 crypto_xor(dst
+ dmreq
->sg_out
.offset
, iv
, cc
->iv_size
);
609 static void crypt_iv_tcw_dtr(struct crypt_config
*cc
)
611 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
613 kzfree(tcw
->iv_seed
);
615 kzfree(tcw
->whitening
);
616 tcw
->whitening
= NULL
;
618 if (tcw
->crc32_tfm
&& !IS_ERR(tcw
->crc32_tfm
))
619 crypto_free_shash(tcw
->crc32_tfm
);
620 tcw
->crc32_tfm
= NULL
;
623 static int crypt_iv_tcw_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
626 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
628 if (cc
->key_size
<= (cc
->iv_size
+ TCW_WHITENING_SIZE
)) {
629 ti
->error
= "Wrong key size for TCW";
633 tcw
->crc32_tfm
= crypto_alloc_shash("crc32", 0, 0);
634 if (IS_ERR(tcw
->crc32_tfm
)) {
635 ti
->error
= "Error initializing CRC32 in TCW";
636 return PTR_ERR(tcw
->crc32_tfm
);
639 tcw
->iv_seed
= kzalloc(cc
->iv_size
, GFP_KERNEL
);
640 tcw
->whitening
= kzalloc(TCW_WHITENING_SIZE
, GFP_KERNEL
);
641 if (!tcw
->iv_seed
|| !tcw
->whitening
) {
642 crypt_iv_tcw_dtr(cc
);
643 ti
->error
= "Error allocating seed storage in TCW";
650 static int crypt_iv_tcw_init(struct crypt_config
*cc
)
652 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
653 int key_offset
= cc
->key_size
- cc
->iv_size
- TCW_WHITENING_SIZE
;
655 memcpy(tcw
->iv_seed
, &cc
->key
[key_offset
], cc
->iv_size
);
656 memcpy(tcw
->whitening
, &cc
->key
[key_offset
+ cc
->iv_size
],
662 static int crypt_iv_tcw_wipe(struct crypt_config
*cc
)
664 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
666 memset(tcw
->iv_seed
, 0, cc
->iv_size
);
667 memset(tcw
->whitening
, 0, TCW_WHITENING_SIZE
);
672 static int crypt_iv_tcw_whitening(struct crypt_config
*cc
,
673 struct dm_crypt_request
*dmreq
,
676 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
677 u64 sector
= cpu_to_le64((u64
)dmreq
->iv_sector
);
678 u8 buf
[TCW_WHITENING_SIZE
];
679 SHASH_DESC_ON_STACK(desc
, tcw
->crc32_tfm
);
682 /* xor whitening with sector number */
683 memcpy(buf
, tcw
->whitening
, TCW_WHITENING_SIZE
);
684 crypto_xor(buf
, (u8
*)§or
, 8);
685 crypto_xor(&buf
[8], (u8
*)§or
, 8);
687 /* calculate crc32 for every 32bit part and xor it */
688 desc
->tfm
= tcw
->crc32_tfm
;
689 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
690 for (i
= 0; i
< 4; i
++) {
691 r
= crypto_shash_init(desc
);
694 r
= crypto_shash_update(desc
, &buf
[i
* 4], 4);
697 r
= crypto_shash_final(desc
, &buf
[i
* 4]);
701 crypto_xor(&buf
[0], &buf
[12], 4);
702 crypto_xor(&buf
[4], &buf
[8], 4);
704 /* apply whitening (8 bytes) to whole sector */
705 for (i
= 0; i
< ((1 << SECTOR_SHIFT
) / 8); i
++)
706 crypto_xor(data
+ i
* 8, buf
, 8);
708 memzero_explicit(buf
, sizeof(buf
));
712 static int crypt_iv_tcw_gen(struct crypt_config
*cc
, u8
*iv
,
713 struct dm_crypt_request
*dmreq
)
715 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
716 u64 sector
= cpu_to_le64((u64
)dmreq
->iv_sector
);
720 /* Remove whitening from ciphertext */
721 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
) {
722 src
= kmap_atomic(sg_page(&dmreq
->sg_in
));
723 r
= crypt_iv_tcw_whitening(cc
, dmreq
, src
+ dmreq
->sg_in
.offset
);
728 memcpy(iv
, tcw
->iv_seed
, cc
->iv_size
);
729 crypto_xor(iv
, (u8
*)§or
, 8);
731 crypto_xor(&iv
[8], (u8
*)§or
, cc
->iv_size
- 8);
736 static int crypt_iv_tcw_post(struct crypt_config
*cc
, u8
*iv
,
737 struct dm_crypt_request
*dmreq
)
742 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
)
745 /* Apply whitening on ciphertext */
746 dst
= kmap_atomic(sg_page(&dmreq
->sg_out
));
747 r
= crypt_iv_tcw_whitening(cc
, dmreq
, dst
+ dmreq
->sg_out
.offset
);
753 static struct crypt_iv_operations crypt_iv_plain_ops
= {
754 .generator
= crypt_iv_plain_gen
757 static struct crypt_iv_operations crypt_iv_plain64_ops
= {
758 .generator
= crypt_iv_plain64_gen
761 static struct crypt_iv_operations crypt_iv_essiv_ops
= {
762 .ctr
= crypt_iv_essiv_ctr
,
763 .dtr
= crypt_iv_essiv_dtr
,
764 .init
= crypt_iv_essiv_init
,
765 .wipe
= crypt_iv_essiv_wipe
,
766 .generator
= crypt_iv_essiv_gen
769 static struct crypt_iv_operations crypt_iv_benbi_ops
= {
770 .ctr
= crypt_iv_benbi_ctr
,
771 .dtr
= crypt_iv_benbi_dtr
,
772 .generator
= crypt_iv_benbi_gen
775 static struct crypt_iv_operations crypt_iv_null_ops
= {
776 .generator
= crypt_iv_null_gen
779 static struct crypt_iv_operations crypt_iv_lmk_ops
= {
780 .ctr
= crypt_iv_lmk_ctr
,
781 .dtr
= crypt_iv_lmk_dtr
,
782 .init
= crypt_iv_lmk_init
,
783 .wipe
= crypt_iv_lmk_wipe
,
784 .generator
= crypt_iv_lmk_gen
,
785 .post
= crypt_iv_lmk_post
788 static struct crypt_iv_operations crypt_iv_tcw_ops
= {
789 .ctr
= crypt_iv_tcw_ctr
,
790 .dtr
= crypt_iv_tcw_dtr
,
791 .init
= crypt_iv_tcw_init
,
792 .wipe
= crypt_iv_tcw_wipe
,
793 .generator
= crypt_iv_tcw_gen
,
794 .post
= crypt_iv_tcw_post
797 static void crypt_convert_init(struct crypt_config
*cc
,
798 struct convert_context
*ctx
,
799 struct bio
*bio_out
, struct bio
*bio_in
,
802 ctx
->bio_in
= bio_in
;
803 ctx
->bio_out
= bio_out
;
805 ctx
->iter_in
= bio_in
->bi_iter
;
807 ctx
->iter_out
= bio_out
->bi_iter
;
808 ctx
->cc_sector
= sector
+ cc
->iv_offset
;
809 init_completion(&ctx
->restart
);
812 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
813 struct ablkcipher_request
*req
)
815 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
818 static struct ablkcipher_request
*req_of_dmreq(struct crypt_config
*cc
,
819 struct dm_crypt_request
*dmreq
)
821 return (struct ablkcipher_request
*)((char *)dmreq
- cc
->dmreq_start
);
824 static u8
*iv_of_dmreq(struct crypt_config
*cc
,
825 struct dm_crypt_request
*dmreq
)
827 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
828 crypto_ablkcipher_alignmask(any_tfm(cc
)) + 1);
831 static int crypt_convert_block(struct crypt_config
*cc
,
832 struct convert_context
*ctx
,
833 struct ablkcipher_request
*req
)
835 struct bio_vec bv_in
= bio_iter_iovec(ctx
->bio_in
, ctx
->iter_in
);
836 struct bio_vec bv_out
= bio_iter_iovec(ctx
->bio_out
, ctx
->iter_out
);
837 struct dm_crypt_request
*dmreq
;
841 dmreq
= dmreq_of_req(cc
, req
);
842 iv
= iv_of_dmreq(cc
, dmreq
);
844 dmreq
->iv_sector
= ctx
->cc_sector
;
846 sg_init_table(&dmreq
->sg_in
, 1);
847 sg_set_page(&dmreq
->sg_in
, bv_in
.bv_page
, 1 << SECTOR_SHIFT
,
850 sg_init_table(&dmreq
->sg_out
, 1);
851 sg_set_page(&dmreq
->sg_out
, bv_out
.bv_page
, 1 << SECTOR_SHIFT
,
854 bio_advance_iter(ctx
->bio_in
, &ctx
->iter_in
, 1 << SECTOR_SHIFT
);
855 bio_advance_iter(ctx
->bio_out
, &ctx
->iter_out
, 1 << SECTOR_SHIFT
);
857 if (cc
->iv_gen_ops
) {
858 r
= cc
->iv_gen_ops
->generator(cc
, iv
, dmreq
);
863 ablkcipher_request_set_crypt(req
, &dmreq
->sg_in
, &dmreq
->sg_out
,
864 1 << SECTOR_SHIFT
, iv
);
866 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
867 r
= crypto_ablkcipher_encrypt(req
);
869 r
= crypto_ablkcipher_decrypt(req
);
871 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
872 r
= cc
->iv_gen_ops
->post(cc
, iv
, dmreq
);
877 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
880 static void crypt_alloc_req(struct crypt_config
*cc
,
881 struct convert_context
*ctx
)
883 unsigned key_index
= ctx
->cc_sector
& (cc
->tfms_count
- 1);
886 ctx
->req
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
888 ablkcipher_request_set_tfm(ctx
->req
, cc
->tfms
[key_index
]);
889 ablkcipher_request_set_callback(ctx
->req
,
890 CRYPTO_TFM_REQ_MAY_BACKLOG
| CRYPTO_TFM_REQ_MAY_SLEEP
,
891 kcryptd_async_done
, dmreq_of_req(cc
, ctx
->req
));
894 static void crypt_free_req(struct crypt_config
*cc
,
895 struct ablkcipher_request
*req
, struct bio
*base_bio
)
897 struct dm_crypt_io
*io
= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
);
899 if ((struct ablkcipher_request
*)(io
+ 1) != req
)
900 mempool_free(req
, cc
->req_pool
);
904 * Encrypt / decrypt data from one bio to another one (can be the same one)
906 static int crypt_convert(struct crypt_config
*cc
,
907 struct convert_context
*ctx
)
911 atomic_set(&ctx
->cc_pending
, 1);
913 while (ctx
->iter_in
.bi_size
&& ctx
->iter_out
.bi_size
) {
915 crypt_alloc_req(cc
, ctx
);
917 atomic_inc(&ctx
->cc_pending
);
919 r
= crypt_convert_block(cc
, ctx
, ctx
->req
);
924 wait_for_completion(&ctx
->restart
);
925 reinit_completion(&ctx
->restart
);
934 atomic_dec(&ctx
->cc_pending
);
941 atomic_dec(&ctx
->cc_pending
);
950 * Generate a new unfragmented bio with the given size
951 * This should never violate the device limitations
952 * May return a smaller bio when running out of pages, indicated by
953 * *out_of_pages set to 1.
955 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
,
956 unsigned *out_of_pages
)
958 struct crypt_config
*cc
= io
->cc
;
960 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
961 gfp_t gfp_mask
= GFP_NOIO
| __GFP_HIGHMEM
;
965 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
969 clone_init(io
, clone
);
972 for (i
= 0; i
< nr_iovecs
; i
++) {
973 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
980 * If additional pages cannot be allocated without waiting,
981 * return a partially-allocated bio. The caller will then try
982 * to allocate more bios while submitting this partial bio.
984 gfp_mask
= (gfp_mask
| __GFP_NOWARN
) & ~__GFP_WAIT
;
986 len
= (size
> PAGE_SIZE
) ? PAGE_SIZE
: size
;
988 if (!bio_add_page(clone
, page
, len
, 0)) {
989 mempool_free(page
, cc
->page_pool
);
996 if (!clone
->bi_iter
.bi_size
) {
1004 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
1009 bio_for_each_segment_all(bv
, clone
, i
) {
1010 BUG_ON(!bv
->bv_page
);
1011 mempool_free(bv
->bv_page
, cc
->page_pool
);
1016 static void crypt_io_init(struct dm_crypt_io
*io
, struct crypt_config
*cc
,
1017 struct bio
*bio
, sector_t sector
)
1021 io
->sector
= sector
;
1025 atomic_set(&io
->io_pending
, 0);
1028 static void crypt_inc_pending(struct dm_crypt_io
*io
)
1030 atomic_inc(&io
->io_pending
);
1034 * One of the bios was finished. Check for completion of
1035 * the whole request and correctly clean up the buffer.
1036 * If base_io is set, wait for the last fragment to complete.
1038 static void crypt_dec_pending(struct dm_crypt_io
*io
)
1040 struct crypt_config
*cc
= io
->cc
;
1041 struct bio
*base_bio
= io
->base_bio
;
1042 struct dm_crypt_io
*base_io
= io
->base_io
;
1043 int error
= io
->error
;
1045 if (!atomic_dec_and_test(&io
->io_pending
))
1049 crypt_free_req(cc
, io
->ctx
.req
, base_bio
);
1050 if (io
!= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
))
1051 mempool_free(io
, cc
->io_pool
);
1053 if (likely(!base_io
))
1054 bio_endio(base_bio
, error
);
1056 if (error
&& !base_io
->error
)
1057 base_io
->error
= error
;
1058 crypt_dec_pending(base_io
);
1063 * kcryptd/kcryptd_io:
1065 * Needed because it would be very unwise to do decryption in an
1066 * interrupt context.
1068 * kcryptd performs the actual encryption or decryption.
1070 * kcryptd_io performs the IO submission.
1072 * They must be separated as otherwise the final stages could be
1073 * starved by new requests which can block in the first stages due
1074 * to memory allocation.
1076 * The work is done per CPU global for all dm-crypt instances.
1077 * They should not depend on each other and do not block.
1079 static void crypt_endio(struct bio
*clone
, int error
)
1081 struct dm_crypt_io
*io
= clone
->bi_private
;
1082 struct crypt_config
*cc
= io
->cc
;
1083 unsigned rw
= bio_data_dir(clone
);
1085 if (unlikely(!bio_flagged(clone
, BIO_UPTODATE
) && !error
))
1089 * free the processed pages
1092 crypt_free_buffer_pages(cc
, clone
);
1096 if (rw
== READ
&& !error
) {
1097 kcryptd_queue_crypt(io
);
1101 if (unlikely(error
))
1104 crypt_dec_pending(io
);
1107 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
1109 struct crypt_config
*cc
= io
->cc
;
1111 clone
->bi_private
= io
;
1112 clone
->bi_end_io
= crypt_endio
;
1113 clone
->bi_bdev
= cc
->dev
->bdev
;
1114 clone
->bi_rw
= io
->base_bio
->bi_rw
;
1117 static int kcryptd_io_read(struct dm_crypt_io
*io
, gfp_t gfp
)
1119 struct crypt_config
*cc
= io
->cc
;
1120 struct bio
*base_bio
= io
->base_bio
;
1124 * The block layer might modify the bvec array, so always
1125 * copy the required bvecs because we need the original
1126 * one in order to decrypt the whole bio data *afterwards*.
1128 clone
= bio_clone_bioset(base_bio
, gfp
, cc
->bs
);
1132 crypt_inc_pending(io
);
1134 clone_init(io
, clone
);
1135 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1137 generic_make_request(clone
);
1141 static void kcryptd_io_write(struct dm_crypt_io
*io
)
1143 struct bio
*clone
= io
->ctx
.bio_out
;
1144 generic_make_request(clone
);
1147 static void kcryptd_io(struct work_struct
*work
)
1149 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1151 if (bio_data_dir(io
->base_bio
) == READ
) {
1152 crypt_inc_pending(io
);
1153 if (kcryptd_io_read(io
, GFP_NOIO
))
1154 io
->error
= -ENOMEM
;
1155 crypt_dec_pending(io
);
1157 kcryptd_io_write(io
);
1160 static void kcryptd_queue_io(struct dm_crypt_io
*io
)
1162 struct crypt_config
*cc
= io
->cc
;
1164 INIT_WORK(&io
->work
, kcryptd_io
);
1165 queue_work(cc
->io_queue
, &io
->work
);
1168 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
, int async
)
1170 struct bio
*clone
= io
->ctx
.bio_out
;
1171 struct crypt_config
*cc
= io
->cc
;
1173 if (unlikely(io
->error
< 0)) {
1174 crypt_free_buffer_pages(cc
, clone
);
1176 crypt_dec_pending(io
);
1180 /* crypt_convert should have filled the clone bio */
1181 BUG_ON(io
->ctx
.iter_out
.bi_size
);
1183 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1186 kcryptd_queue_io(io
);
1188 generic_make_request(clone
);
1191 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
1193 struct crypt_config
*cc
= io
->cc
;
1195 struct dm_crypt_io
*new_io
;
1197 unsigned out_of_pages
= 0;
1198 unsigned remaining
= io
->base_bio
->bi_iter
.bi_size
;
1199 sector_t sector
= io
->sector
;
1203 * Prevent io from disappearing until this function completes.
1205 crypt_inc_pending(io
);
1206 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
1209 * The allocated buffers can be smaller than the whole bio,
1210 * so repeat the whole process until all the data can be handled.
1213 clone
= crypt_alloc_buffer(io
, remaining
, &out_of_pages
);
1214 if (unlikely(!clone
)) {
1215 io
->error
= -ENOMEM
;
1219 io
->ctx
.bio_out
= clone
;
1220 io
->ctx
.iter_out
= clone
->bi_iter
;
1222 remaining
-= clone
->bi_iter
.bi_size
;
1223 sector
+= bio_sectors(clone
);
1225 crypt_inc_pending(io
);
1227 r
= crypt_convert(cc
, &io
->ctx
);
1231 crypt_finished
= atomic_dec_and_test(&io
->ctx
.cc_pending
);
1233 /* Encryption was already finished, submit io now */
1234 if (crypt_finished
) {
1235 kcryptd_crypt_write_io_submit(io
, 0);
1238 * If there was an error, do not try next fragments.
1239 * For async, error is processed in async handler.
1241 if (unlikely(r
< 0))
1244 io
->sector
= sector
;
1248 * Out of memory -> run queues
1249 * But don't wait if split was due to the io size restriction
1251 if (unlikely(out_of_pages
))
1252 congestion_wait(BLK_RW_ASYNC
, HZ
/100);
1255 * With async crypto it is unsafe to share the crypto context
1256 * between fragments, so switch to a new dm_crypt_io structure.
1258 if (unlikely(!crypt_finished
&& remaining
)) {
1259 new_io
= mempool_alloc(cc
->io_pool
, GFP_NOIO
);
1260 crypt_io_init(new_io
, io
->cc
, io
->base_bio
, sector
);
1261 crypt_inc_pending(new_io
);
1262 crypt_convert_init(cc
, &new_io
->ctx
, NULL
,
1263 io
->base_bio
, sector
);
1264 new_io
->ctx
.iter_in
= io
->ctx
.iter_in
;
1267 * Fragments after the first use the base_io
1271 new_io
->base_io
= io
;
1273 new_io
->base_io
= io
->base_io
;
1274 crypt_inc_pending(io
->base_io
);
1275 crypt_dec_pending(io
);
1282 crypt_dec_pending(io
);
1285 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
)
1287 crypt_dec_pending(io
);
1290 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
1292 struct crypt_config
*cc
= io
->cc
;
1295 crypt_inc_pending(io
);
1297 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
1300 r
= crypt_convert(cc
, &io
->ctx
);
1304 if (atomic_dec_and_test(&io
->ctx
.cc_pending
))
1305 kcryptd_crypt_read_done(io
);
1307 crypt_dec_pending(io
);
1310 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1313 struct dm_crypt_request
*dmreq
= async_req
->data
;
1314 struct convert_context
*ctx
= dmreq
->ctx
;
1315 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1316 struct crypt_config
*cc
= io
->cc
;
1318 if (error
== -EINPROGRESS
) {
1319 complete(&ctx
->restart
);
1323 if (!error
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1324 error
= cc
->iv_gen_ops
->post(cc
, iv_of_dmreq(cc
, dmreq
), dmreq
);
1329 crypt_free_req(cc
, req_of_dmreq(cc
, dmreq
), io
->base_bio
);
1331 if (!atomic_dec_and_test(&ctx
->cc_pending
))
1334 if (bio_data_dir(io
->base_bio
) == READ
)
1335 kcryptd_crypt_read_done(io
);
1337 kcryptd_crypt_write_io_submit(io
, 1);
1340 static void kcryptd_crypt(struct work_struct
*work
)
1342 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1344 if (bio_data_dir(io
->base_bio
) == READ
)
1345 kcryptd_crypt_read_convert(io
);
1347 kcryptd_crypt_write_convert(io
);
1350 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
1352 struct crypt_config
*cc
= io
->cc
;
1354 INIT_WORK(&io
->work
, kcryptd_crypt
);
1355 queue_work(cc
->crypt_queue
, &io
->work
);
1359 * Decode key from its hex representation
1361 static int crypt_decode_key(u8
*key
, char *hex
, unsigned int size
)
1368 for (i
= 0; i
< size
; i
++) {
1372 if (kstrtou8(buffer
, 16, &key
[i
]))
1382 static void crypt_free_tfms(struct crypt_config
*cc
)
1389 for (i
= 0; i
< cc
->tfms_count
; i
++)
1390 if (cc
->tfms
[i
] && !IS_ERR(cc
->tfms
[i
])) {
1391 crypto_free_ablkcipher(cc
->tfms
[i
]);
1399 static int crypt_alloc_tfms(struct crypt_config
*cc
, char *ciphermode
)
1404 cc
->tfms
= kmalloc(cc
->tfms_count
* sizeof(struct crypto_ablkcipher
*),
1409 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1410 cc
->tfms
[i
] = crypto_alloc_ablkcipher(ciphermode
, 0, 0);
1411 if (IS_ERR(cc
->tfms
[i
])) {
1412 err
= PTR_ERR(cc
->tfms
[i
]);
1413 crypt_free_tfms(cc
);
1421 static int crypt_setkey_allcpus(struct crypt_config
*cc
)
1423 unsigned subkey_size
;
1426 /* Ignore extra keys (which are used for IV etc) */
1427 subkey_size
= (cc
->key_size
- cc
->key_extra_size
) >> ilog2(cc
->tfms_count
);
1429 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1430 r
= crypto_ablkcipher_setkey(cc
->tfms
[i
],
1431 cc
->key
+ (i
* subkey_size
),
1440 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
1443 int key_string_len
= strlen(key
);
1445 /* The key size may not be changed. */
1446 if (cc
->key_size
!= (key_string_len
>> 1))
1449 /* Hyphen (which gives a key_size of zero) means there is no key. */
1450 if (!cc
->key_size
&& strcmp(key
, "-"))
1453 if (cc
->key_size
&& crypt_decode_key(cc
->key
, key
, cc
->key_size
) < 0)
1456 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1458 r
= crypt_setkey_allcpus(cc
);
1461 /* Hex key string not needed after here, so wipe it. */
1462 memset(key
, '0', key_string_len
);
1467 static int crypt_wipe_key(struct crypt_config
*cc
)
1469 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1470 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
1472 return crypt_setkey_allcpus(cc
);
1475 static void crypt_dtr(struct dm_target
*ti
)
1477 struct crypt_config
*cc
= ti
->private;
1485 destroy_workqueue(cc
->io_queue
);
1486 if (cc
->crypt_queue
)
1487 destroy_workqueue(cc
->crypt_queue
);
1489 crypt_free_tfms(cc
);
1492 bioset_free(cc
->bs
);
1495 mempool_destroy(cc
->page_pool
);
1497 mempool_destroy(cc
->req_pool
);
1499 mempool_destroy(cc
->io_pool
);
1501 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
1502 cc
->iv_gen_ops
->dtr(cc
);
1505 dm_put_device(ti
, cc
->dev
);
1508 kzfree(cc
->cipher_string
);
1510 /* Must zero key material before freeing */
1514 static int crypt_ctr_cipher(struct dm_target
*ti
,
1515 char *cipher_in
, char *key
)
1517 struct crypt_config
*cc
= ti
->private;
1518 char *tmp
, *cipher
, *chainmode
, *ivmode
, *ivopts
, *keycount
;
1519 char *cipher_api
= NULL
;
1523 /* Convert to crypto api definition? */
1524 if (strchr(cipher_in
, '(')) {
1525 ti
->error
= "Bad cipher specification";
1529 cc
->cipher_string
= kstrdup(cipher_in
, GFP_KERNEL
);
1530 if (!cc
->cipher_string
)
1534 * Legacy dm-crypt cipher specification
1535 * cipher[:keycount]-mode-iv:ivopts
1538 keycount
= strsep(&tmp
, "-");
1539 cipher
= strsep(&keycount
, ":");
1543 else if (sscanf(keycount
, "%u%c", &cc
->tfms_count
, &dummy
) != 1 ||
1544 !is_power_of_2(cc
->tfms_count
)) {
1545 ti
->error
= "Bad cipher key count specification";
1548 cc
->key_parts
= cc
->tfms_count
;
1549 cc
->key_extra_size
= 0;
1551 cc
->cipher
= kstrdup(cipher
, GFP_KERNEL
);
1555 chainmode
= strsep(&tmp
, "-");
1556 ivopts
= strsep(&tmp
, "-");
1557 ivmode
= strsep(&ivopts
, ":");
1560 DMWARN("Ignoring unexpected additional cipher options");
1563 * For compatibility with the original dm-crypt mapping format, if
1564 * only the cipher name is supplied, use cbc-plain.
1566 if (!chainmode
|| (!strcmp(chainmode
, "plain") && !ivmode
)) {
1571 if (strcmp(chainmode
, "ecb") && !ivmode
) {
1572 ti
->error
= "IV mechanism required";
1576 cipher_api
= kmalloc(CRYPTO_MAX_ALG_NAME
, GFP_KERNEL
);
1580 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
1581 "%s(%s)", chainmode
, cipher
);
1587 /* Allocate cipher */
1588 ret
= crypt_alloc_tfms(cc
, cipher_api
);
1590 ti
->error
= "Error allocating crypto tfm";
1595 cc
->iv_size
= crypto_ablkcipher_ivsize(any_tfm(cc
));
1597 /* at least a 64 bit sector number should fit in our buffer */
1598 cc
->iv_size
= max(cc
->iv_size
,
1599 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
1601 DMWARN("Selected cipher does not support IVs");
1605 /* Choose ivmode, see comments at iv code. */
1607 cc
->iv_gen_ops
= NULL
;
1608 else if (strcmp(ivmode
, "plain") == 0)
1609 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
1610 else if (strcmp(ivmode
, "plain64") == 0)
1611 cc
->iv_gen_ops
= &crypt_iv_plain64_ops
;
1612 else if (strcmp(ivmode
, "essiv") == 0)
1613 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
1614 else if (strcmp(ivmode
, "benbi") == 0)
1615 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
1616 else if (strcmp(ivmode
, "null") == 0)
1617 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
1618 else if (strcmp(ivmode
, "lmk") == 0) {
1619 cc
->iv_gen_ops
= &crypt_iv_lmk_ops
;
1621 * Version 2 and 3 is recognised according
1622 * to length of provided multi-key string.
1623 * If present (version 3), last key is used as IV seed.
1624 * All keys (including IV seed) are always the same size.
1626 if (cc
->key_size
% cc
->key_parts
) {
1628 cc
->key_extra_size
= cc
->key_size
/ cc
->key_parts
;
1630 } else if (strcmp(ivmode
, "tcw") == 0) {
1631 cc
->iv_gen_ops
= &crypt_iv_tcw_ops
;
1632 cc
->key_parts
+= 2; /* IV + whitening */
1633 cc
->key_extra_size
= cc
->iv_size
+ TCW_WHITENING_SIZE
;
1636 ti
->error
= "Invalid IV mode";
1640 /* Initialize and set key */
1641 ret
= crypt_set_key(cc
, key
);
1643 ti
->error
= "Error decoding and setting key";
1648 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
) {
1649 ret
= cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
);
1651 ti
->error
= "Error creating IV";
1656 /* Initialize IV (set keys for ESSIV etc) */
1657 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
) {
1658 ret
= cc
->iv_gen_ops
->init(cc
);
1660 ti
->error
= "Error initialising IV";
1671 ti
->error
= "Cannot allocate cipher strings";
1676 * Construct an encryption mapping:
1677 * <cipher> <key> <iv_offset> <dev_path> <start>
1679 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
1681 struct crypt_config
*cc
;
1682 unsigned int key_size
, opt_params
;
1683 unsigned long long tmpll
;
1685 size_t iv_size_padding
;
1686 struct dm_arg_set as
;
1687 const char *opt_string
;
1690 static struct dm_arg _args
[] = {
1691 {0, 1, "Invalid number of feature args"},
1695 ti
->error
= "Not enough arguments";
1699 key_size
= strlen(argv
[1]) >> 1;
1701 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
1703 ti
->error
= "Cannot allocate encryption context";
1706 cc
->key_size
= key_size
;
1709 ret
= crypt_ctr_cipher(ti
, argv
[0], argv
[1]);
1714 cc
->io_pool
= mempool_create_slab_pool(MIN_IOS
, _crypt_io_pool
);
1716 ti
->error
= "Cannot allocate crypt io mempool";
1720 cc
->dmreq_start
= sizeof(struct ablkcipher_request
);
1721 cc
->dmreq_start
+= crypto_ablkcipher_reqsize(any_tfm(cc
));
1722 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, __alignof__(struct dm_crypt_request
));
1724 if (crypto_ablkcipher_alignmask(any_tfm(cc
)) < CRYPTO_MINALIGN
) {
1725 /* Allocate the padding exactly */
1726 iv_size_padding
= -(cc
->dmreq_start
+ sizeof(struct dm_crypt_request
))
1727 & crypto_ablkcipher_alignmask(any_tfm(cc
));
1730 * If the cipher requires greater alignment than kmalloc
1731 * alignment, we don't know the exact position of the
1732 * initialization vector. We must assume worst case.
1734 iv_size_padding
= crypto_ablkcipher_alignmask(any_tfm(cc
));
1737 cc
->req_pool
= mempool_create_kmalloc_pool(MIN_IOS
, cc
->dmreq_start
+
1738 sizeof(struct dm_crypt_request
) + iv_size_padding
+ cc
->iv_size
);
1739 if (!cc
->req_pool
) {
1740 ti
->error
= "Cannot allocate crypt request mempool";
1744 cc
->per_bio_data_size
= ti
->per_bio_data_size
=
1745 ALIGN(sizeof(struct dm_crypt_io
) + cc
->dmreq_start
+
1746 sizeof(struct dm_crypt_request
) + iv_size_padding
+ cc
->iv_size
,
1747 ARCH_KMALLOC_MINALIGN
);
1749 cc
->page_pool
= mempool_create_page_pool(MIN_POOL_PAGES
, 0);
1750 if (!cc
->page_pool
) {
1751 ti
->error
= "Cannot allocate page mempool";
1755 cc
->bs
= bioset_create(MIN_IOS
, 0);
1757 ti
->error
= "Cannot allocate crypt bioset";
1762 if (sscanf(argv
[2], "%llu%c", &tmpll
, &dummy
) != 1) {
1763 ti
->error
= "Invalid iv_offset sector";
1766 cc
->iv_offset
= tmpll
;
1768 if (dm_get_device(ti
, argv
[3], dm_table_get_mode(ti
->table
), &cc
->dev
)) {
1769 ti
->error
= "Device lookup failed";
1773 if (sscanf(argv
[4], "%llu%c", &tmpll
, &dummy
) != 1) {
1774 ti
->error
= "Invalid device sector";
1782 /* Optional parameters */
1787 ret
= dm_read_arg_group(_args
, &as
, &opt_params
, &ti
->error
);
1791 opt_string
= dm_shift_arg(&as
);
1793 if (opt_params
== 1 && opt_string
&&
1794 !strcasecmp(opt_string
, "allow_discards"))
1795 ti
->num_discard_bios
= 1;
1796 else if (opt_params
) {
1798 ti
->error
= "Invalid feature arguments";
1804 cc
->io_queue
= alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM
, 1);
1805 if (!cc
->io_queue
) {
1806 ti
->error
= "Couldn't create kcryptd io queue";
1810 cc
->crypt_queue
= alloc_workqueue("kcryptd",
1811 WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
, 1);
1812 if (!cc
->crypt_queue
) {
1813 ti
->error
= "Couldn't create kcryptd queue";
1817 ti
->num_flush_bios
= 1;
1818 ti
->discard_zeroes_data_unsupported
= true;
1827 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
)
1829 struct dm_crypt_io
*io
;
1830 struct crypt_config
*cc
= ti
->private;
1833 * If bio is REQ_FLUSH or REQ_DISCARD, just bypass crypt queues.
1834 * - for REQ_FLUSH device-mapper core ensures that no IO is in-flight
1835 * - for REQ_DISCARD caller must use flush if IO ordering matters
1837 if (unlikely(bio
->bi_rw
& (REQ_FLUSH
| REQ_DISCARD
))) {
1838 bio
->bi_bdev
= cc
->dev
->bdev
;
1839 if (bio_sectors(bio
))
1840 bio
->bi_iter
.bi_sector
= cc
->start
+
1841 dm_target_offset(ti
, bio
->bi_iter
.bi_sector
);
1842 return DM_MAPIO_REMAPPED
;
1845 io
= dm_per_bio_data(bio
, cc
->per_bio_data_size
);
1846 crypt_io_init(io
, cc
, bio
, dm_target_offset(ti
, bio
->bi_iter
.bi_sector
));
1847 io
->ctx
.req
= (struct ablkcipher_request
*)(io
+ 1);
1849 if (bio_data_dir(io
->base_bio
) == READ
) {
1850 if (kcryptd_io_read(io
, GFP_NOWAIT
))
1851 kcryptd_queue_io(io
);
1853 kcryptd_queue_crypt(io
);
1855 return DM_MAPIO_SUBMITTED
;
1858 static void crypt_status(struct dm_target
*ti
, status_type_t type
,
1859 unsigned status_flags
, char *result
, unsigned maxlen
)
1861 struct crypt_config
*cc
= ti
->private;
1865 case STATUSTYPE_INFO
:
1869 case STATUSTYPE_TABLE
:
1870 DMEMIT("%s ", cc
->cipher_string
);
1872 if (cc
->key_size
> 0)
1873 for (i
= 0; i
< cc
->key_size
; i
++)
1874 DMEMIT("%02x", cc
->key
[i
]);
1878 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
1879 cc
->dev
->name
, (unsigned long long)cc
->start
);
1881 if (ti
->num_discard_bios
)
1882 DMEMIT(" 1 allow_discards");
1888 static void crypt_postsuspend(struct dm_target
*ti
)
1890 struct crypt_config
*cc
= ti
->private;
1892 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1895 static int crypt_preresume(struct dm_target
*ti
)
1897 struct crypt_config
*cc
= ti
->private;
1899 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
1900 DMERR("aborting resume - crypt key is not set.");
1907 static void crypt_resume(struct dm_target
*ti
)
1909 struct crypt_config
*cc
= ti
->private;
1911 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1914 /* Message interface
1918 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
1920 struct crypt_config
*cc
= ti
->private;
1926 if (!strcasecmp(argv
[0], "key")) {
1927 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
1928 DMWARN("not suspended during key manipulation.");
1931 if (argc
== 3 && !strcasecmp(argv
[1], "set")) {
1932 ret
= crypt_set_key(cc
, argv
[2]);
1935 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
)
1936 ret
= cc
->iv_gen_ops
->init(cc
);
1939 if (argc
== 2 && !strcasecmp(argv
[1], "wipe")) {
1940 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->wipe
) {
1941 ret
= cc
->iv_gen_ops
->wipe(cc
);
1945 return crypt_wipe_key(cc
);
1950 DMWARN("unrecognised message received.");
1954 static int crypt_merge(struct dm_target
*ti
, struct bvec_merge_data
*bvm
,
1955 struct bio_vec
*biovec
, int max_size
)
1957 struct crypt_config
*cc
= ti
->private;
1958 struct request_queue
*q
= bdev_get_queue(cc
->dev
->bdev
);
1960 if (!q
->merge_bvec_fn
)
1963 bvm
->bi_bdev
= cc
->dev
->bdev
;
1964 bvm
->bi_sector
= cc
->start
+ dm_target_offset(ti
, bvm
->bi_sector
);
1966 return min(max_size
, q
->merge_bvec_fn(q
, bvm
, biovec
));
1969 static int crypt_iterate_devices(struct dm_target
*ti
,
1970 iterate_devices_callout_fn fn
, void *data
)
1972 struct crypt_config
*cc
= ti
->private;
1974 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
1977 static struct target_type crypt_target
= {
1979 .version
= {1, 13, 0},
1980 .module
= THIS_MODULE
,
1984 .status
= crypt_status
,
1985 .postsuspend
= crypt_postsuspend
,
1986 .preresume
= crypt_preresume
,
1987 .resume
= crypt_resume
,
1988 .message
= crypt_message
,
1989 .merge
= crypt_merge
,
1990 .iterate_devices
= crypt_iterate_devices
,
1993 static int __init
dm_crypt_init(void)
1997 _crypt_io_pool
= KMEM_CACHE(dm_crypt_io
, 0);
1998 if (!_crypt_io_pool
)
2001 r
= dm_register_target(&crypt_target
);
2003 DMERR("register failed %d", r
);
2004 kmem_cache_destroy(_crypt_io_pool
);
2010 static void __exit
dm_crypt_exit(void)
2012 dm_unregister_target(&crypt_target
);
2013 kmem_cache_destroy(_crypt_io_pool
);
2016 module_init(dm_crypt_init
);
2017 module_exit(dm_crypt_exit
);
2019 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
2020 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
2021 MODULE_LICENSE("GPL");