arch/arm/crypto/ghash-ce-glue.c

   1 /*
   2  * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
   3  *
   4  * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  */
  10
  11 #include <asm/hwcap.h>
  12 #include <asm/neon.h>
  13 #include <asm/simd.h>
  14 #include <asm/unaligned.h>
  15 #include <crypto/cryptd.h>
  16 #include <crypto/internal/hash.h>
  17 #include <crypto/gf128mul.h>
  18 #include <linux/crypto.h>
  19 #include <linux/module.h>
  20
  21 MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
  22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  23 MODULE_LICENSE("GPL v2");
  24
  25 #define GHASH_BLOCK_SIZE        16
  26 #define GHASH_DIGEST_SIZE       16
  27
  28 struct ghash_key {
  29         u64     a;
  30         u64     b;
  31 };
  32
  33 struct ghash_desc_ctx {
  34         u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
  35         u8 buf[GHASH_BLOCK_SIZE];
  36         u32 count;
  37 };
  38
  39 struct ghash_async_ctx {
  40         struct cryptd_ahash *cryptd_tfm;
  41 };
  42
  43 asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
  44                                    struct ghash_key const *k, const char *head);
  45
  46 static int ghash_init(struct shash_desc *desc)
  47 {
  48         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  49
  50         *ctx = (struct ghash_desc_ctx){};
  51         return 0;
  52 }
  53
  54 static int ghash_update(struct shash_desc *desc, const u8 *src,
  55                         unsigned int len)
  56 {
  57         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  58         unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  59
  60         ctx->count += len;
  61
  62         if ((partial + len) >= GHASH_BLOCK_SIZE) {
  63                 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  64                 int blocks;
  65
  66                 if (partial) {
  67                         int p = GHASH_BLOCK_SIZE - partial;
  68
  69                         memcpy(ctx->buf + partial, src, p);
  70                         src += p;
  71                         len -= p;
  72                 }
  73
  74                 blocks = len / GHASH_BLOCK_SIZE;
  75                 len %= GHASH_BLOCK_SIZE;
  76
  77                 kernel_neon_begin();
  78                 pmull_ghash_update(blocks, ctx->digest, src, key,
  79                                    partial ? ctx->buf : NULL);
  80                 kernel_neon_end();
  81                 src += blocks * GHASH_BLOCK_SIZE;
  82                 partial = 0;
  83         }
  84         if (len)
  85                 memcpy(ctx->buf + partial, src, len);
  86         return 0;
  87 }
  88
  89 static int ghash_final(struct shash_desc *desc, u8 *dst)
  90 {
  91         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  92         unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  93
  94         if (partial) {
  95                 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  96
  97                 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
  98                 kernel_neon_begin();
  99                 pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
 100                 kernel_neon_end();
 101         }
 102         put_unaligned_be64(ctx->digest[1], dst);
 103         put_unaligned_be64(ctx->digest[0], dst + 8);
 104
 105         *ctx = (struct ghash_desc_ctx){};
 106         return 0;
 107 }
 108
 109 static int ghash_setkey(struct crypto_shash *tfm,
 110                         const u8 *inkey, unsigned int keylen)
 111 {
 112         struct ghash_key *key = crypto_shash_ctx(tfm);
 113         u64 a, b;
 114
 115         if (keylen != GHASH_BLOCK_SIZE) {
 116                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 117                 return -EINVAL;
 118         }
 119
 120         /* perform multiplication by 'x' in GF(2^128) */
 121         b = get_unaligned_be64(inkey);
 122         a = get_unaligned_be64(inkey + 8);
 123
 124         key->a = (a << 1) | (b >> 63);
 125         key->b = (b << 1) | (a >> 63);
 126
 127         if (b >> 63)
 128                 key->b ^= 0xc200000000000000UL;
 129
 130         return 0;
 131 }
 132
 133 static struct shash_alg ghash_alg = {
 134         .digestsize             = GHASH_DIGEST_SIZE,
 135         .init                   = ghash_init,
 136         .update                 = ghash_update,
 137         .final                  = ghash_final,
 138         .setkey                 = ghash_setkey,
 139         .descsize               = sizeof(struct ghash_desc_ctx),
 140         .base                   = {
 141                 .cra_name       = "__ghash",
 142                 .cra_driver_name = "__driver-ghash-ce",
 143                 .cra_priority   = 0,
 144                 .cra_flags      = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL,
 145                 .cra_blocksize  = GHASH_BLOCK_SIZE,
 146                 .cra_ctxsize    = sizeof(struct ghash_key),
 147                 .cra_module     = THIS_MODULE,
 148         },
 149 };
 150
 151 static int ghash_async_init(struct ahash_request *req)
 152 {
 153         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 154         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 155         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 156         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 157         struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 158         struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
 159
 160         desc->tfm = child;
 161         desc->flags = req->base.flags;
 162         return crypto_shash_init(desc);
 163 }
 164
 165 static int ghash_async_update(struct ahash_request *req)
 166 {
 167         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 168         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 169         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 170         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 171
 172         if (!may_use_simd() ||
 173             (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
 174                 memcpy(cryptd_req, req, sizeof(*req));
 175                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 176                 return crypto_ahash_update(cryptd_req);
 177         } else {
 178                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 179                 return shash_ahash_update(req, desc);
 180         }
 181 }
 182
 183 static int ghash_async_final(struct ahash_request *req)
 184 {
 185         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 186         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 187         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 188         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 189
 190         if (!may_use_simd() ||
 191             (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
 192                 memcpy(cryptd_req, req, sizeof(*req));
 193                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 194                 return crypto_ahash_final(cryptd_req);
 195         } else {
 196                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 197                 return crypto_shash_final(desc, req->result);
 198         }
 199 }
 200
 201 static int ghash_async_digest(struct ahash_request *req)
 202 {
 203         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 204         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 205         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 206         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 207
 208         if (!may_use_simd() ||
 209             (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
 210                 memcpy(cryptd_req, req, sizeof(*req));
 211                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 212                 return crypto_ahash_digest(cryptd_req);
 213         } else {
 214                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 215                 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
 216
 217                 desc->tfm = child;
 218                 desc->flags = req->base.flags;
 219                 return shash_ahash_digest(req, desc);
 220         }
 221 }
 222
 223 static int ghash_async_import(struct ahash_request *req, const void *in)
 224 {
 225         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 226         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 227         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 228         struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 229
 230         desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm);
 231         desc->flags = req->base.flags;
 232
 233         return crypto_shash_import(desc, in);
 234 }
 235
 236 static int ghash_async_export(struct ahash_request *req, void *out)
 237 {
 238         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 239         struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 240
 241         return crypto_shash_export(desc, out);
 242 }
 243
 244 static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
 245                               unsigned int keylen)
 246 {
 247         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 248         struct crypto_ahash *child = &ctx->cryptd_tfm->base;
 249         int err;
 250
 251         crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 252         crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
 253                                & CRYPTO_TFM_REQ_MASK);
 254         err = crypto_ahash_setkey(child, key, keylen);
 255         crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
 256                                & CRYPTO_TFM_RES_MASK);
 257
 258         return err;
 259 }
 260
 261 static int ghash_async_init_tfm(struct crypto_tfm *tfm)
 262 {
 263         struct cryptd_ahash *cryptd_tfm;
 264         struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
 265
 266         cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce",
 267                                         CRYPTO_ALG_INTERNAL,
 268                                         CRYPTO_ALG_INTERNAL);
 269         if (IS_ERR(cryptd_tfm))
 270                 return PTR_ERR(cryptd_tfm);
 271         ctx->cryptd_tfm = cryptd_tfm;
 272         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 273                                  sizeof(struct ahash_request) +
 274                                  crypto_ahash_reqsize(&cryptd_tfm->base));
 275
 276         return 0;
 277 }
 278
 279 static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
 280 {
 281         struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
 282
 283         cryptd_free_ahash(ctx->cryptd_tfm);
 284 }
 285
 286 static struct ahash_alg ghash_async_alg = {
 287         .init                   = ghash_async_init,
 288         .update                 = ghash_async_update,
 289         .final                  = ghash_async_final,
 290         .setkey                 = ghash_async_setkey,
 291         .digest                 = ghash_async_digest,
 292         .import                 = ghash_async_import,
 293         .export                 = ghash_async_export,
 294         .halg.digestsize        = GHASH_DIGEST_SIZE,
 295         .halg.statesize         = sizeof(struct ghash_desc_ctx),
 296         .halg.base              = {
 297                 .cra_name       = "ghash",
 298                 .cra_driver_name = "ghash-ce",
 299                 .cra_priority   = 300,
 300                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
 301                 .cra_blocksize  = GHASH_BLOCK_SIZE,
 302                 .cra_type       = &crypto_ahash_type,
 303                 .cra_ctxsize    = sizeof(struct ghash_async_ctx),
 304                 .cra_module     = THIS_MODULE,
 305                 .cra_init       = ghash_async_init_tfm,
 306                 .cra_exit       = ghash_async_exit_tfm,
 307         },
 308 };
 309
 310 static int __init ghash_ce_mod_init(void)
 311 {
 312         int err;
 313
 314         if (!(elf_hwcap2 & HWCAP2_PMULL))
 315                 return -ENODEV;
 316
 317         err = crypto_register_shash(&ghash_alg);
 318         if (err)
 319                 return err;
 320         err = crypto_register_ahash(&ghash_async_alg);
 321         if (err)
 322                 goto err_shash;
 323
 324         return 0;
 325
 326 err_shash:
 327         crypto_unregister_shash(&ghash_alg);
 328         return err;
 329 }
 330
 331 static void __exit ghash_ce_mod_exit(void)
 332 {
 333         crypto_unregister_ahash(&ghash_async_alg);
 334         crypto_unregister_shash(&ghash_alg);
 335 }
 336
 337 module_init(ghash_ce_mod_init);
 338 module_exit(ghash_ce_mod_exit);