arch/arm/crypto/aesbs-glue.c

   1 /*
   2  * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES
   3  *
   4  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/neon.h>
  12 #include <crypto/aes.h>
  13 #include <crypto/ablk_helper.h>
  14 #include <crypto/algapi.h>
  15 #include <linux/module.h>
  16
  17 #include "aes_glue.h"
  18
  19 #define BIT_SLICED_KEY_MAXSIZE  (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE)
  20
  21 struct BS_KEY {
  22         struct AES_KEY  rk;
  23         int             converted;
  24         u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE];
  25 } __aligned(8);
  26
  27 asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in);
  28 asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in);
  29
  30 asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes,
  31                                   struct BS_KEY *key, u8 iv[]);
  32
  33 asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks,
  34                                            struct BS_KEY *key, u8 const iv[]);
  35
  36 asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes,
  37                                   struct BS_KEY *key, u8 tweak[]);
  38
  39 asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes,
  40                                   struct BS_KEY *key, u8 tweak[]);
  41
  42 struct aesbs_cbc_ctx {
  43         struct AES_KEY  enc;
  44         struct BS_KEY   dec;
  45 };
  46
  47 struct aesbs_ctr_ctx {
  48         struct BS_KEY   enc;
  49 };
  50
  51 struct aesbs_xts_ctx {
  52         struct BS_KEY   enc;
  53         struct BS_KEY   dec;
  54         struct AES_KEY  twkey;
  55 };
  56
  57 static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  58                              unsigned int key_len)
  59 {
  60         struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
  61         int bits = key_len * 8;
  62
  63         if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
  64                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  65                 return -EINVAL;
  66         }
  67         ctx->dec.rk = ctx->enc;
  68         private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
  69         ctx->dec.converted = 0;
  70         return 0;
  71 }
  72
  73 static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  74                              unsigned int key_len)
  75 {
  76         struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
  77         int bits = key_len * 8;
  78
  79         if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
  80                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  81                 return -EINVAL;
  82         }
  83         ctx->enc.converted = 0;
  84         return 0;
  85 }
  86
  87 static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  88                              unsigned int key_len)
  89 {
  90         struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  91         int bits = key_len * 4;
  92
  93         if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
  94                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  95                 return -EINVAL;
  96         }
  97         ctx->dec.rk = ctx->enc.rk;
  98         private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
  99         private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey);
 100         ctx->enc.converted = ctx->dec.converted = 0;
 101         return 0;
 102 }
 103
 104 static int aesbs_cbc_encrypt(struct blkcipher_desc *desc,
 105                              struct scatterlist *dst,
 106                              struct scatterlist *src, unsigned int nbytes)
 107 {
 108         struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 109         struct blkcipher_walk walk;
 110         int err;
 111
 112         blkcipher_walk_init(&walk, dst, src, nbytes);
 113         err = blkcipher_walk_virt(desc, &walk);
 114
 115         while (walk.nbytes) {
 116                 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
 117                 u8 *src = walk.src.virt.addr;
 118
 119                 if (walk.dst.virt.addr == walk.src.virt.addr) {
 120                         u8 *iv = walk.iv;
 121
 122                         do {
 123                                 crypto_xor(src, iv, AES_BLOCK_SIZE);
 124                                 AES_encrypt(src, src, &ctx->enc);
 125                                 iv = src;
 126                                 src += AES_BLOCK_SIZE;
 127                         } while (--blocks);
 128                         memcpy(walk.iv, iv, AES_BLOCK_SIZE);
 129                 } else {
 130                         u8 *dst = walk.dst.virt.addr;
 131
 132                         do {
 133                                 crypto_xor(walk.iv, src, AES_BLOCK_SIZE);
 134                                 AES_encrypt(walk.iv, dst, &ctx->enc);
 135                                 memcpy(walk.iv, dst, AES_BLOCK_SIZE);
 136                                 src += AES_BLOCK_SIZE;
 137                                 dst += AES_BLOCK_SIZE;
 138                         } while (--blocks);
 139                 }
 140                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 141         }
 142         return err;
 143 }
 144
 145 static int aesbs_cbc_decrypt(struct blkcipher_desc *desc,
 146                              struct scatterlist *dst,
 147                              struct scatterlist *src, unsigned int nbytes)
 148 {
 149         struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 150         struct blkcipher_walk walk;
 151         int err;
 152
 153         blkcipher_walk_init(&walk, dst, src, nbytes);
 154         err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
 155
 156         while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) {
 157                 kernel_neon_begin();
 158                 bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
 159                                   walk.nbytes, &ctx->dec, walk.iv);
 160                 kernel_neon_end();
 161                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 162         }
 163         while (walk.nbytes) {
 164                 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
 165                 u8 *dst = walk.dst.virt.addr;
 166                 u8 *src = walk.src.virt.addr;
 167                 u8 bk[2][AES_BLOCK_SIZE];
 168                 u8 *iv = walk.iv;
 169
 170                 do {
 171                         if (walk.dst.virt.addr == walk.src.virt.addr)
 172                                 memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE);
 173
 174                         AES_decrypt(src, dst, &ctx->dec.rk);
 175                         crypto_xor(dst, iv, AES_BLOCK_SIZE);
 176
 177                         if (walk.dst.virt.addr == walk.src.virt.addr)
 178                                 iv = bk[blocks & 1];
 179                         else
 180                                 iv = src;
 181
 182                         dst += AES_BLOCK_SIZE;
 183                         src += AES_BLOCK_SIZE;
 184                 } while (--blocks);
 185                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 186         }
 187         return err;
 188 }
 189
 190 static void inc_be128_ctr(__be32 ctr[], u32 addend)
 191 {
 192         int i;
 193
 194         for (i = 3; i >= 0; i--, addend = 1) {
 195                 u32 n = be32_to_cpu(ctr[i]) + addend;
 196
 197                 ctr[i] = cpu_to_be32(n);
 198                 if (n >= addend)
 199                         break;
 200         }
 201 }
 202
 203 static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
 204                              struct scatterlist *dst, struct scatterlist *src,
 205                              unsigned int nbytes)
 206 {
 207         struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 208         struct blkcipher_walk walk;
 209         u32 blocks;
 210         int err;
 211
 212         blkcipher_walk_init(&walk, dst, src, nbytes);
 213         err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
 214
 215         while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
 216                 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
 217                 __be32 *ctr = (__be32 *)walk.iv;
 218                 u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
 219
 220                 /* avoid 32 bit counter overflow in the NEON code */
 221                 if (unlikely(headroom < blocks)) {
 222                         blocks = headroom + 1;
 223                         tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
 224                 }
 225                 kernel_neon_begin();
 226                 bsaes_ctr32_encrypt_blocks(walk.src.virt.addr,
 227                                            walk.dst.virt.addr, blocks,
 228                                            &ctx->enc, walk.iv);
 229                 kernel_neon_end();
 230                 inc_be128_ctr(ctr, blocks);
 231
 232                 nbytes -= blocks * AES_BLOCK_SIZE;
 233                 if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE)
 234                         break;
 235
 236                 err = blkcipher_walk_done(desc, &walk, tail);
 237         }
 238         if (walk.nbytes) {
 239                 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
 240                 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 241                 u8 ks[AES_BLOCK_SIZE];
 242
 243                 AES_encrypt(walk.iv, ks, &ctx->enc.rk);
 244                 if (tdst != tsrc)
 245                         memcpy(tdst, tsrc, nbytes);
 246                 crypto_xor(tdst, ks, nbytes);
 247                 err = blkcipher_walk_done(desc, &walk, 0);
 248         }
 249         return err;
 250 }
 251
 252 static int aesbs_xts_encrypt(struct blkcipher_desc *desc,
 253                              struct scatterlist *dst,
 254                              struct scatterlist *src, unsigned int nbytes)
 255 {
 256         struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 257         struct blkcipher_walk walk;
 258         int err;
 259
 260         blkcipher_walk_init(&walk, dst, src, nbytes);
 261         err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
 262
 263         /* generate the initial tweak */
 264         AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
 265
 266         while (walk.nbytes) {
 267                 kernel_neon_begin();
 268                 bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
 269                                   walk.nbytes, &ctx->enc, walk.iv);
 270                 kernel_neon_end();
 271                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 272         }
 273         return err;
 274 }
 275
 276 static int aesbs_xts_decrypt(struct blkcipher_desc *desc,
 277                              struct scatterlist *dst,
 278                              struct scatterlist *src, unsigned int nbytes)
 279 {
 280         struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 281         struct blkcipher_walk walk;
 282         int err;
 283
 284         blkcipher_walk_init(&walk, dst, src, nbytes);
 285         err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
 286
 287         /* generate the initial tweak */
 288         AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
 289
 290         while (walk.nbytes) {
 291                 kernel_neon_begin();
 292                 bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
 293                                   walk.nbytes, &ctx->dec, walk.iv);
 294                 kernel_neon_end();
 295                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 296         }
 297         return err;
 298 }
 299
 300 static struct crypto_alg aesbs_algs[] = { {
 301         .cra_name               = "__cbc-aes-neonbs",
 302         .cra_driver_name        = "__driver-cbc-aes-neonbs",
 303         .cra_priority           = 0,
 304         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 305                                   CRYPTO_ALG_INTERNAL,
 306         .cra_blocksize          = AES_BLOCK_SIZE,
 307         .cra_ctxsize            = sizeof(struct aesbs_cbc_ctx),
 308         .cra_alignmask          = 7,
 309         .cra_type               = &crypto_blkcipher_type,
 310         .cra_module             = THIS_MODULE,
 311         .cra_blkcipher = {
 312                 .min_keysize    = AES_MIN_KEY_SIZE,
 313                 .max_keysize    = AES_MAX_KEY_SIZE,
 314                 .ivsize         = AES_BLOCK_SIZE,
 315                 .setkey         = aesbs_cbc_set_key,
 316                 .encrypt        = aesbs_cbc_encrypt,
 317                 .decrypt        = aesbs_cbc_decrypt,
 318         },
 319 }, {
 320         .cra_name               = "__ctr-aes-neonbs",
 321         .cra_driver_name        = "__driver-ctr-aes-neonbs",
 322         .cra_priority           = 0,
 323         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 324                                   CRYPTO_ALG_INTERNAL,
 325         .cra_blocksize          = 1,
 326         .cra_ctxsize            = sizeof(struct aesbs_ctr_ctx),
 327         .cra_alignmask          = 7,
 328         .cra_type               = &crypto_blkcipher_type,
 329         .cra_module             = THIS_MODULE,
 330         .cra_blkcipher = {
 331                 .min_keysize    = AES_MIN_KEY_SIZE,
 332                 .max_keysize    = AES_MAX_KEY_SIZE,
 333                 .ivsize         = AES_BLOCK_SIZE,
 334                 .setkey         = aesbs_ctr_set_key,
 335                 .encrypt        = aesbs_ctr_encrypt,
 336                 .decrypt        = aesbs_ctr_encrypt,
 337         },
 338 }, {
 339         .cra_name               = "__xts-aes-neonbs",
 340         .cra_driver_name        = "__driver-xts-aes-neonbs",
 341         .cra_priority           = 0,
 342         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 343                                   CRYPTO_ALG_INTERNAL,
 344         .cra_blocksize          = AES_BLOCK_SIZE,
 345         .cra_ctxsize            = sizeof(struct aesbs_xts_ctx),
 346         .cra_alignmask          = 7,
 347         .cra_type               = &crypto_blkcipher_type,
 348         .cra_module             = THIS_MODULE,
 349         .cra_blkcipher = {
 350                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 351                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 352                 .ivsize         = AES_BLOCK_SIZE,
 353                 .setkey         = aesbs_xts_set_key,
 354                 .encrypt        = aesbs_xts_encrypt,
 355                 .decrypt        = aesbs_xts_decrypt,
 356         },
 357 }, {
 358         .cra_name               = "cbc(aes)",
 359         .cra_driver_name        = "cbc-aes-neonbs",
 360         .cra_priority           = 300,
 361         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 362         .cra_blocksize          = AES_BLOCK_SIZE,
 363         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 364         .cra_alignmask          = 7,
 365         .cra_type               = &crypto_ablkcipher_type,
 366         .cra_module             = THIS_MODULE,
 367         .cra_init               = ablk_init,
 368         .cra_exit               = ablk_exit,
 369         .cra_ablkcipher = {
 370                 .min_keysize    = AES_MIN_KEY_SIZE,
 371                 .max_keysize    = AES_MAX_KEY_SIZE,
 372                 .ivsize         = AES_BLOCK_SIZE,
 373                 .setkey         = ablk_set_key,
 374                 .encrypt        = __ablk_encrypt,
 375                 .decrypt        = ablk_decrypt,
 376         }
 377 }, {
 378         .cra_name               = "ctr(aes)",
 379         .cra_driver_name        = "ctr-aes-neonbs",
 380         .cra_priority           = 300,
 381         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 382         .cra_blocksize          = 1,
 383         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 384         .cra_alignmask          = 7,
 385         .cra_type               = &crypto_ablkcipher_type,
 386         .cra_module             = THIS_MODULE,
 387         .cra_init               = ablk_init,
 388         .cra_exit               = ablk_exit,
 389         .cra_ablkcipher = {
 390                 .min_keysize    = AES_MIN_KEY_SIZE,
 391                 .max_keysize    = AES_MAX_KEY_SIZE,
 392                 .ivsize         = AES_BLOCK_SIZE,
 393                 .setkey         = ablk_set_key,
 394                 .encrypt        = ablk_encrypt,
 395                 .decrypt        = ablk_decrypt,
 396         }
 397 }, {
 398         .cra_name               = "xts(aes)",
 399         .cra_driver_name        = "xts-aes-neonbs",
 400         .cra_priority           = 300,
 401         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 402         .cra_blocksize          = AES_BLOCK_SIZE,
 403         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 404         .cra_alignmask          = 7,
 405         .cra_type               = &crypto_ablkcipher_type,
 406         .cra_module             = THIS_MODULE,
 407         .cra_init               = ablk_init,
 408         .cra_exit               = ablk_exit,
 409         .cra_ablkcipher = {
 410                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 411                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 412                 .ivsize         = AES_BLOCK_SIZE,
 413                 .setkey         = ablk_set_key,
 414                 .encrypt        = ablk_encrypt,
 415                 .decrypt        = ablk_decrypt,
 416         }
 417 } };
 418
 419 static int __init aesbs_mod_init(void)
 420 {
 421         if (!cpu_has_neon())
 422                 return -ENODEV;
 423
 424         return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
 425 }
 426
 427 static void __exit aesbs_mod_exit(void)
 428 {
 429         crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
 430 }
 431
 432 module_init(aesbs_mod_init);
 433 module_exit(aesbs_mod_exit);
 434
 435 MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON");
 436 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 437 MODULE_LICENSE("GPL");