projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm64: Clean up the default pgprot setting
[deliverable/linux.git] / drivers / crypto / n2_core.c
CommitLineData
0a625fd2
DM		 1/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
2 *
eb7caf35 3 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
0a625fd2
DM		 4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_device.h>
12#include <linux/cpumask.h>
13#include <linux/slab.h>
14#include <linux/interrupt.h>
15#include <linux/crypto.h>
16#include <crypto/md5.h>
17#include <crypto/sha.h>
18#include <crypto/aes.h>
19#include <crypto/des.h>
20#include <linux/mutex.h>
21#include <linux/delay.h>
22#include <linux/sched.h>
23
24#include <crypto/internal/hash.h>
25#include <crypto/scatterwalk.h>
26#include <crypto/algapi.h>
27
28#include <asm/hypervisor.h>
29#include <asm/mdesc.h>
30
31#include "n2_core.h"
32
33#define DRV_MODULE_NAME "n2_crypto"
eb7caf35
DM		 34#define DRV_MODULE_VERSION "0.2"
35#define DRV_MODULE_RELDATE "July 28, 2011"
0a625fd2 36
49cfe4db 37static char version[] =
0a625fd2
DM		 38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
39
40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41MODULE_DESCRIPTION("Niagara2 Crypto driver");
42MODULE_LICENSE("GPL");
43MODULE_VERSION(DRV_MODULE_VERSION);
44
10803624 45#define N2_CRA_PRIORITY 200
0a625fd2
DM		 46
47static DEFINE_MUTEX(spu_lock);
48
49struct spu_queue {
50 cpumask_t sharing;
51 unsigned long qhandle;
52
53 spinlock_t lock;
54 u8 q_type;
55 void *q;
56 unsigned long head;
57 unsigned long tail;
58 struct list_head jobs;
59
60 unsigned long devino;
61
62 char irq_name[32];
63 unsigned int irq;
64
65 struct list_head list;
66};
67
68static struct spu_queue **cpu_to_cwq;
69static struct spu_queue **cpu_to_mau;
70
71static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
72{
73 if (q->q_type == HV_NCS_QTYPE_MAU) {
74 off += MAU_ENTRY_SIZE;
75 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
76 off = 0;
77 } else {
78 off += CWQ_ENTRY_SIZE;
79 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
80 off = 0;
81 }
82 return off;
83}
84
85struct n2_request_common {
86 struct list_head entry;
87 unsigned int offset;
88};
89#define OFFSET_NOT_RUNNING (~(unsigned int)0)
90
91/* An async job request records the final tail value it used in
92 * n2_request_common->offset, test to see if that offset is in
93 * the range old_head, new_head, inclusive.
94 */
95static inline bool job_finished(struct spu_queue *q, unsigned int offset,
96 unsigned long old_head, unsigned long new_head)
97{
98 if (old_head <= new_head) {
99 if (offset > old_head && offset <= new_head)
100 return true;
101 } else {
102 if (offset > old_head || offset <= new_head)
103 return true;
104 }
105 return false;
106}
107
108/* When the HEAD marker is unequal to the actual HEAD, we get
109 * a virtual device INO interrupt. We should process the
110 * completed CWQ entries and adjust the HEAD marker to clear
111 * the IRQ.
112 */
113static irqreturn_t cwq_intr(int irq, void *dev_id)
114{
115 unsigned long off, new_head, hv_ret;
116 struct spu_queue *q = dev_id;
117
118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119 smp_processor_id(), q->qhandle);
120
121 spin_lock(&q->lock);
122
123 hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
124
125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126 smp_processor_id(), new_head, hv_ret);
127
128 for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
129 /* XXX ... XXX */
130 }
131
132 hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
133 if (hv_ret == HV_EOK)
134 q->head = new_head;
135
136 spin_unlock(&q->lock);
137
138 return IRQ_HANDLED;
139}
140
141static irqreturn_t mau_intr(int irq, void *dev_id)
142{
143 struct spu_queue *q = dev_id;
144 unsigned long head, hv_ret;
145
146 spin_lock(&q->lock);
147
148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149 smp_processor_id(), q->qhandle);
150
151 hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
152
153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154 smp_processor_id(), head, hv_ret);
155
156 sun4v_ncs_sethead_marker(q->qhandle, head);
157
158 spin_unlock(&q->lock);
159
160 return IRQ_HANDLED;
161}
162
163static void *spu_queue_next(struct spu_queue *q, void *cur)
164{
165 return q->q + spu_next_offset(q, cur - q->q);
166}
167
168static int spu_queue_num_free(struct spu_queue *q)
169{
170 unsigned long head = q->head;
171 unsigned long tail = q->tail;
172 unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
173 unsigned long diff;
174
175 if (head > tail)
176 diff = head - tail;
177 else
178 diff = (end - tail) + head;
179
180 return (diff / CWQ_ENTRY_SIZE) - 1;
181}
182
183static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
184{
185 int avail = spu_queue_num_free(q);
186
187 if (avail >= num_entries)
188 return q->q + q->tail;
189
190 return NULL;
191}
192
193static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
194{
195 unsigned long hv_ret, new_tail;
196
197 new_tail = spu_next_offset(q, last - q->q);
198
199 hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
200 if (hv_ret == HV_EOK)
201 q->tail = new_tail;
202 return hv_ret;
203}
204
205static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
206 int enc_type, int auth_type,
207 unsigned int hash_len,
208 bool sfas, bool sob, bool eob, bool encrypt,
209 int opcode)
210{
211 u64 word = (len - 1) & CONTROL_LEN;
212
213 word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
214 word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
215 word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
216 if (sfas)
217 word |= CONTROL_STORE_FINAL_AUTH_STATE;
218 if (sob)
219 word |= CONTROL_START_OF_BLOCK;
220 if (eob)
221 word |= CONTROL_END_OF_BLOCK;
222 if (encrypt)
223 word |= CONTROL_ENCRYPT;
224 if (hmac_key_len)
225 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
226 if (hash_len)
227 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
228
229 return word;
230}
231
232#if 0
233static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
234{
235 if (this_len >= 64 ||
236 qp->head != qp->tail)
237 return true;
238 return false;
239}
240#endif
241
3a2c0346
DM		 242struct n2_ahash_alg {
243 struct list_head entry;
244 const char *hash_zero;
245 const u32 *hash_init;
246 u8 hw_op_hashsz;
247 u8 digest_size;
248 u8 auth_type;
dc4ccfd1 249 u8 hmac_type;
3a2c0346
DM		 250 struct ahash_alg alg;
251};
252
253static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
254{
255 struct crypto_alg *alg = tfm->__crt_alg;
256 struct ahash_alg *ahash_alg;
257
258 ahash_alg = container_of(alg, struct ahash_alg, halg.base);
259
260 return container_of(ahash_alg, struct n2_ahash_alg, alg);
261}
262
dc4ccfd1
DM		 263struct n2_hmac_alg {
264 const char *child_alg;
265 struct n2_ahash_alg derived;
266};
267
268static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
269{
270 struct crypto_alg *alg = tfm->__crt_alg;
271 struct ahash_alg *ahash_alg;
272
273 ahash_alg = container_of(alg, struct ahash_alg, halg.base);
274
275 return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
276}
277
0a625fd2 278struct n2_hash_ctx {
c9aa55e5
DM		 279 struct crypto_ahash *fallback_tfm;
280};
0a625fd2 281
dc4ccfd1
DM		 282#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */
283
284struct n2_hmac_ctx {
285 struct n2_hash_ctx base;
286
287 struct crypto_shash *child_shash;
288
289 int hash_key_len;
290 unsigned char hash_key[N2_HASH_KEY_MAX];
291};
292
c9aa55e5 293struct n2_hash_req_ctx {
0a625fd2
DM		 294 union {
295 struct md5_state md5;
296 struct sha1_state sha1;
297 struct sha256_state sha256;
298 } u;
299
c9aa55e5 300 struct ahash_request fallback_req;
0a625fd2
DM		 301};
302
303static int n2_hash_async_init(struct ahash_request *req)
304{
c9aa55e5 305 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
0a625fd2
DM		 306 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
307 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
308
c9aa55e5
DM		 309 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
310 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
0a625fd2 311
c9aa55e5 312 return crypto_ahash_init(&rctx->fallback_req);
0a625fd2
DM		 313}
314
315static int n2_hash_async_update(struct ahash_request *req)
316{
c9aa55e5 317 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
0a625fd2
DM		 318 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
319 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
320
c9aa55e5
DM		 321 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
322 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
323 rctx->fallback_req.nbytes = req->nbytes;
324 rctx->fallback_req.src = req->src;
0a625fd2 325
c9aa55e5 326 return crypto_ahash_update(&rctx->fallback_req);
0a625fd2
DM		 327}
328
329static int n2_hash_async_final(struct ahash_request *req)
330{
c9aa55e5 331 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
0a625fd2
DM		 332 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
333 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
334
c9aa55e5
DM		 335 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
336 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
337 rctx->fallback_req.result = req->result;
0a625fd2 338
c9aa55e5 339 return crypto_ahash_final(&rctx->fallback_req);
0a625fd2
DM		 340}
341
342static int n2_hash_async_finup(struct ahash_request *req)
343{
c9aa55e5 344 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
0a625fd2
DM		 345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
346 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
347
c9aa55e5
DM		 348 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
349 rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
350 rctx->fallback_req.nbytes = req->nbytes;
351 rctx->fallback_req.src = req->src;
352 rctx->fallback_req.result = req->result;
0a625fd2 353
c9aa55e5 354 return crypto_ahash_finup(&rctx->fallback_req);
0a625fd2
DM		 355}
356
357static int n2_hash_cra_init(struct crypto_tfm *tfm)
358{
359 const char *fallback_driver_name = tfm->__crt_alg->cra_name;
360 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
361 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
362 struct crypto_ahash *fallback_tfm;
363 int err;
364
365 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
366 CRYPTO_ALG_NEED_FALLBACK);
367 if (IS_ERR(fallback_tfm)) {
368 pr_warning("Fallback driver '%s' could not be loaded!\n",
369 fallback_driver_name);
370 err = PTR_ERR(fallback_tfm);
371 goto out;
372 }
373
c9aa55e5
DM		 374 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
375 crypto_ahash_reqsize(fallback_tfm)));
376
377 ctx->fallback_tfm = fallback_tfm;
0a625fd2
DM		 378 return 0;
379
380out:
381 return err;
382}
383
384static void n2_hash_cra_exit(struct crypto_tfm *tfm)
385{
386 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
387 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
388
c9aa55e5 389 crypto_free_ahash(ctx->fallback_tfm);
0a625fd2
DM		 390}
391
dc4ccfd1
DM		 392static int n2_hmac_cra_init(struct crypto_tfm *tfm)
393{
394 const char *fallback_driver_name = tfm->__crt_alg->cra_name;
395 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
396 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
397 struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
398 struct crypto_ahash *fallback_tfm;
399 struct crypto_shash *child_shash;
400 int err;
401
402 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
403 CRYPTO_ALG_NEED_FALLBACK);
404 if (IS_ERR(fallback_tfm)) {
405 pr_warning("Fallback driver '%s' could not be loaded!\n",
406 fallback_driver_name);
407 err = PTR_ERR(fallback_tfm);
408 goto out;
409 }
410
411 child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
412 if (IS_ERR(child_shash)) {
413 pr_warning("Child shash '%s' could not be loaded!\n",
414 n2alg->child_alg);
415 err = PTR_ERR(child_shash);
416 goto out_free_fallback;
417 }
418
419 crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
420 crypto_ahash_reqsize(fallback_tfm)));
421
422 ctx->child_shash = child_shash;
423 ctx->base.fallback_tfm = fallback_tfm;
424 return 0;
425
426out_free_fallback:
427 crypto_free_ahash(fallback_tfm);
428
429out:
430 return err;
431}
432
433static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
434{
435 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
436 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
437
438 crypto_free_ahash(ctx->base.fallback_tfm);
439 crypto_free_shash(ctx->child_shash);
440}
441
442static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
443 unsigned int keylen)
444{
445 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
446 struct crypto_shash *child_shash = ctx->child_shash;
447 struct crypto_ahash *fallback_tfm;
448 struct {
449 struct shash_desc shash;
450 char ctx[crypto_shash_descsize(child_shash)];
451 } desc;
452 int err, bs, ds;
453
454 fallback_tfm = ctx->base.fallback_tfm;
455 err = crypto_ahash_setkey(fallback_tfm, key, keylen);
456 if (err)
457 return err;
458
459 desc.shash.tfm = child_shash;
460 desc.shash.flags = crypto_ahash_get_flags(tfm) &
461 CRYPTO_TFM_REQ_MAY_SLEEP;
462
463 bs = crypto_shash_blocksize(child_shash);
464 ds = crypto_shash_digestsize(child_shash);
465 BUG_ON(ds > N2_HASH_KEY_MAX);
466 if (keylen > bs) {
467 err = crypto_shash_digest(&desc.shash, key, keylen,
468 ctx->hash_key);
469 if (err)
470 return err;
471 keylen = ds;
472 } else if (keylen <= N2_HASH_KEY_MAX)
473 memcpy(ctx->hash_key, key, keylen);
474
475 ctx->hash_key_len = keylen;
476
477 return err;
478}
479
0a625fd2
DM		 480static unsigned long wait_for_tail(struct spu_queue *qp)
481{
482 unsigned long head, hv_ret;
483
484 do {
485 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
486 if (hv_ret != HV_EOK) {
487 pr_err("Hypervisor error on gethead\n");
488 break;
489 }
490 if (head == qp->tail) {
491 qp->head = head;
492 break;
493 }
494 } while (1);
495 return hv_ret;
496}
497
498static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
499 struct cwq_initial_entry *ent)
500{
501 unsigned long hv_ret = spu_queue_submit(qp, ent);
502
503 if (hv_ret == HV_EOK)
504 hv_ret = wait_for_tail(qp);
505
506 return hv_ret;
507}
508
3a2c0346
DM		 509static int n2_do_async_digest(struct ahash_request *req,
510 unsigned int auth_type, unsigned int digest_size,
dc4ccfd1
DM		 511 unsigned int result_size, void *hash_loc,
512 unsigned long auth_key, unsigned int auth_key_len)
0a625fd2
DM		 513{
514 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0a625fd2
DM		 515 struct cwq_initial_entry *ent;
516 struct crypto_hash_walk walk;
517 struct spu_queue *qp;
518 unsigned long flags;
519 int err = -ENODEV;
520 int nbytes, cpu;
521
522 /* The total effective length of the operation may not
523 * exceed 2^16.
524 */
525 if (unlikely(req->nbytes > (1 << 16))) {
c9aa55e5 526 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
65a23d67 527 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
c9aa55e5
DM		 528
529 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
530 rctx->fallback_req.base.flags =
0a625fd2 531 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
c9aa55e5
DM		 532 rctx->fallback_req.nbytes = req->nbytes;
533 rctx->fallback_req.src = req->src;
534 rctx->fallback_req.result = req->result;
0a625fd2 535
c9aa55e5 536 return crypto_ahash_digest(&rctx->fallback_req);
0a625fd2
DM		 537 }
538
0a625fd2
DM		 539 nbytes = crypto_hash_walk_first(req, &walk);
540
541 cpu = get_cpu();
542 qp = cpu_to_cwq[cpu];
543 if (!qp)
544 goto out;
545
546 spin_lock_irqsave(&qp->lock, flags);
547
548 /* XXX can do better, improve this later by doing a by-hand scatterlist
549 * XXX walk, etc.
550 */
551 ent = qp->q + qp->tail;
552
dc4ccfd1 553 ent->control = control_word_base(nbytes, auth_key_len, 0,
0a625fd2
DM		 554 auth_type, digest_size,
555 false, true, false, false,
556 OPCODE_INPLACE_BIT |
557 OPCODE_AUTH_MAC);
558 ent->src_addr = __pa(walk.data);
dc4ccfd1 559 ent->auth_key_addr = auth_key;
0a625fd2
DM		 560 ent->auth_iv_addr = __pa(hash_loc);
561 ent->final_auth_state_addr = 0UL;
562 ent->enc_key_addr = 0UL;
563 ent->enc_iv_addr = 0UL;
564 ent->dest_addr = __pa(hash_loc);
565
566 nbytes = crypto_hash_walk_done(&walk, 0);
567 while (nbytes > 0) {
568 ent = spu_queue_next(qp, ent);
569
570 ent->control = (nbytes - 1);
571 ent->src_addr = __pa(walk.data);
572 ent->auth_key_addr = 0UL;
573 ent->auth_iv_addr = 0UL;
574 ent->final_auth_state_addr = 0UL;
575 ent->enc_key_addr = 0UL;
576 ent->enc_iv_addr = 0UL;
577 ent->dest_addr = 0UL;
578
579 nbytes = crypto_hash_walk_done(&walk, 0);
580 }
581 ent->control |= CONTROL_END_OF_BLOCK;
582
583 if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
584 err = -EINVAL;
585 else
586 err = 0;
587
588 spin_unlock_irqrestore(&qp->lock, flags);
589
590 if (!err)
591 memcpy(req->result, hash_loc, result_size);
592out:
593 put_cpu();
594
595 return err;
596}
597
3a2c0346 598static int n2_hash_async_digest(struct ahash_request *req)
0a625fd2 599{
3a2c0346 600 struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
c9aa55e5 601 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
3a2c0346 602 int ds;
0a625fd2 603
3a2c0346 604 ds = n2alg->digest_size;
0a625fd2 605 if (unlikely(req->nbytes == 0)) {
3a2c0346 606 memcpy(req->result, n2alg->hash_zero, ds);
0a625fd2
DM		 607 return 0;
608 }
3a2c0346 609 memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
0a625fd2 610
3a2c0346
DM		 611 return n2_do_async_digest(req, n2alg->auth_type,
612 n2alg->hw_op_hashsz, ds,
dc4ccfd1
DM		 613 &rctx->u, 0UL, 0);
614}
615
616static int n2_hmac_async_digest(struct ahash_request *req)
617{
618 struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
619 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
620 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
621 struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
622 int ds;
623
624 ds = n2alg->derived.digest_size;
625 if (unlikely(req->nbytes == 0) ||
626 unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
627 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
628 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
629
630 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
631 rctx->fallback_req.base.flags =
632 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
633 rctx->fallback_req.nbytes = req->nbytes;
634 rctx->fallback_req.src = req->src;
635 rctx->fallback_req.result = req->result;
636
637 return crypto_ahash_digest(&rctx->fallback_req);
638 }
639 memcpy(&rctx->u, n2alg->derived.hash_init,
640 n2alg->derived.hw_op_hashsz);
641
642 return n2_do_async_digest(req, n2alg->derived.hmac_type,
643 n2alg->derived.hw_op_hashsz, ds,
644 &rctx->u,
645 __pa(&ctx->hash_key),
646 ctx->hash_key_len);
0a625fd2
DM		 647}
648
649struct n2_cipher_context {
650 int key_len;
651 int enc_type;
652 union {
653 u8 aes[AES_MAX_KEY_SIZE];
654 u8 des[DES_KEY_SIZE];
655 u8 des3[3 * DES_KEY_SIZE];
656 u8 arc4[258]; /* S-box, X, Y */
657 } key;
658};
659
660#define N2_CHUNK_ARR_LEN 16
661
662struct n2_crypto_chunk {
663 struct list_head entry;
664 unsigned long iv_paddr : 44;
665 unsigned long arr_len : 20;
666 unsigned long dest_paddr;
667 unsigned long dest_final;
668 struct {
669 unsigned long src_paddr : 44;
670 unsigned long src_len : 20;
671 } arr[N2_CHUNK_ARR_LEN];
672};
673
674struct n2_request_context {
675 struct ablkcipher_walk walk;
676 struct list_head chunk_list;
677 struct n2_crypto_chunk chunk;
678 u8 temp_iv[16];
679};
680
681/* The SPU allows some level of flexibility for partial cipher blocks
682 * being specified in a descriptor.
683 *
684 * It merely requires that every descriptor's length field is at least
685 * as large as the cipher block size. This means that a cipher block
686 * can span at most 2 descriptors. However, this does not allow a
687 * partial block to span into the final descriptor as that would
688 * violate the rule (since every descriptor's length must be at lest
689 * the block size). So, for example, assuming an 8 byte block size:
690 *
691 * 0xe --> 0xa --> 0x8
692 *
693 * is a valid length sequence, whereas:
694 *
695 * 0xe --> 0xb --> 0x7
696 *
697 * is not a valid sequence.
698 */
699
700struct n2_cipher_alg {
701 struct list_head entry;
702 u8 enc_type;
703 struct crypto_alg alg;
704};
705
706static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
707{
708 struct crypto_alg *alg = tfm->__crt_alg;
709
710 return container_of(alg, struct n2_cipher_alg, alg);
711}
712
713struct n2_cipher_request_context {
714 struct ablkcipher_walk walk;
715};
716
717static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
718 unsigned int keylen)
719{
720 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
721 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
722 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
723
724 ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
725
726 switch (keylen) {
727 case AES_KEYSIZE_128:
728 ctx->enc_type |= ENC_TYPE_ALG_AES128;
729 break;
730 case AES_KEYSIZE_192:
731 ctx->enc_type |= ENC_TYPE_ALG_AES192;
732 break;
733 case AES_KEYSIZE_256:
734 ctx->enc_type |= ENC_TYPE_ALG_AES256;
735 break;
736 default:
737 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
738 return -EINVAL;
739 }
740
741 ctx->key_len = keylen;
742 memcpy(ctx->key.aes, key, keylen);
743 return 0;
744}
745
746static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
747 unsigned int keylen)
748{
749 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
750 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
751 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
752 u32 tmp[DES_EXPKEY_WORDS];
753 int err;
754
755 ctx->enc_type = n2alg->enc_type;
756
757 if (keylen != DES_KEY_SIZE) {
758 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
759 return -EINVAL;
760 }
761
762 err = des_ekey(tmp, key);
763 if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
764 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
765 return -EINVAL;
766 }
767
768 ctx->key_len = keylen;
769 memcpy(ctx->key.des, key, keylen);
770 return 0;
771}
772
773static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
774 unsigned int keylen)
775{
776 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
777 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
778 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
779
780 ctx->enc_type = n2alg->enc_type;
781
782 if (keylen != (3 * DES_KEY_SIZE)) {
783 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
784 return -EINVAL;
785 }
786 ctx->key_len = keylen;
787 memcpy(ctx->key.des3, key, keylen);
788 return 0;
789}
790
791static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
792 unsigned int keylen)
793{
794 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
795 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
796 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
797 u8 *s = ctx->key.arc4;
798 u8 *x = s + 256;
799 u8 *y = x + 1;
800 int i, j, k;
801
802 ctx->enc_type = n2alg->enc_type;
803
804 j = k = 0;
805 *x = 0;
806 *y = 0;
807 for (i = 0; i < 256; i++)
808 s[i] = i;
809 for (i = 0; i < 256; i++) {
810 u8 a = s[i];
811 j = (j + key[k] + a) & 0xff;
812 s[i] = s[j];
813 s[j] = a;
814 if (++k >= keylen)
815 k = 0;
816 }
817
818 return 0;
819}
820
821static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
822{
823 int this_len = nbytes;
824
825 this_len -= (nbytes & (block_size - 1));
826 return this_len > (1 << 16) ? (1 << 16) : this_len;
827}
828
829static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
830 struct spu_queue *qp, bool encrypt)
831{
832 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
833 struct cwq_initial_entry *ent;
834 bool in_place;
835 int i;
836
837 ent = spu_queue_alloc(qp, cp->arr_len);
838 if (!ent) {
839 pr_info("queue_alloc() of %d fails\n",
840 cp->arr_len);
841 return -EBUSY;
842 }
843
844 in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
845
846 ent->control = control_word_base(cp->arr[0].src_len,
847 0, ctx->enc_type, 0, 0,
848 false, true, false, encrypt,
849 OPCODE_ENCRYPT |
850 (in_place ? OPCODE_INPLACE_BIT : 0));
851 ent->src_addr = cp->arr[0].src_paddr;
852 ent->auth_key_addr = 0UL;
853 ent->auth_iv_addr = 0UL;
854 ent->final_auth_state_addr = 0UL;
855 ent->enc_key_addr = __pa(&ctx->key);
856 ent->enc_iv_addr = cp->iv_paddr;
857 ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
858
859 for (i = 1; i < cp->arr_len; i++) {
860 ent = spu_queue_next(qp, ent);
861
862 ent->control = cp->arr[i].src_len - 1;
863 ent->src_addr = cp->arr[i].src_paddr;
864 ent->auth_key_addr = 0UL;
865 ent->auth_iv_addr = 0UL;
866 ent->final_auth_state_addr = 0UL;
867 ent->enc_key_addr = 0UL;
868 ent->enc_iv_addr = 0UL;
869 ent->dest_addr = 0UL;
870 }
871 ent->control |= CONTROL_END_OF_BLOCK;
872
873 return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
874}
875
876static int n2_compute_chunks(struct ablkcipher_request *req)
877{
878 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
879 struct ablkcipher_walk *walk = &rctx->walk;
880 struct n2_crypto_chunk *chunk;
881 unsigned long dest_prev;
882 unsigned int tot_len;
883 bool prev_in_place;
884 int err, nbytes;
885
886 ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
887 err = ablkcipher_walk_phys(req, walk);
888 if (err)
889 return err;
890
891 INIT_LIST_HEAD(&rctx->chunk_list);
892
893 chunk = &rctx->chunk;
894 INIT_LIST_HEAD(&chunk->entry);
895
896 chunk->iv_paddr = 0UL;
897 chunk->arr_len = 0;
898 chunk->dest_paddr = 0UL;
899
900 prev_in_place = false;
901 dest_prev = ~0UL;
902 tot_len = 0;
903
904 while ((nbytes = walk->nbytes) != 0) {
905 unsigned long dest_paddr, src_paddr;
906 bool in_place;
907 int this_len;
908
909 src_paddr = (page_to_phys(walk->src.page) +
910 walk->src.offset);
911 dest_paddr = (page_to_phys(walk->dst.page) +
912 walk->dst.offset);
913 in_place = (src_paddr == dest_paddr);
914 this_len = cipher_descriptor_len(nbytes, walk->blocksize);
915
916 if (chunk->arr_len != 0) {
917 if (in_place != prev_in_place ||
918 (!prev_in_place &&
919 dest_paddr != dest_prev) ||
920 chunk->arr_len == N2_CHUNK_ARR_LEN ||
921 tot_len + this_len > (1 << 16)) {
922 chunk->dest_final = dest_prev;
923 list_add_tail(&chunk->entry,
924 &rctx->chunk_list);
925 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
926 if (!chunk) {
927 err = -ENOMEM;
928 break;
929 }
930 INIT_LIST_HEAD(&chunk->entry);
931 }
932 }
933 if (chunk->arr_len == 0) {
934 chunk->dest_paddr = dest_paddr;
935 tot_len = 0;
936 }
937 chunk->arr[chunk->arr_len].src_paddr = src_paddr;
938 chunk->arr[chunk->arr_len].src_len = this_len;
939 chunk->arr_len++;
940
941 dest_prev = dest_paddr + this_len;
942 prev_in_place = in_place;
943 tot_len += this_len;
944
945 err = ablkcipher_walk_done(req, walk, nbytes - this_len);
946 if (err)
947 break;
948 }
949 if (!err && chunk->arr_len != 0) {
950 chunk->dest_final = dest_prev;
951 list_add_tail(&chunk->entry, &rctx->chunk_list);
952 }
953
954 return err;
955}
956
957static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
958{
959 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
960 struct n2_crypto_chunk *c, *tmp;
961
962 if (final_iv)
963 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
964
965 ablkcipher_walk_complete(&rctx->walk);
966 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
967 list_del(&c->entry);
968 if (unlikely(c != &rctx->chunk))
969 kfree(c);
970 }
971
972}
973
974static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
975{
976 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
977 struct crypto_tfm *tfm = req->base.tfm;
978 int err = n2_compute_chunks(req);
979 struct n2_crypto_chunk *c, *tmp;
980 unsigned long flags, hv_ret;
981 struct spu_queue *qp;
982
983 if (err)
984 return err;
985
986 qp = cpu_to_cwq[get_cpu()];
987 err = -ENODEV;
988 if (!qp)
989 goto out;
990
991 spin_lock_irqsave(&qp->lock, flags);
992
993 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
994 err = __n2_crypt_chunk(tfm, c, qp, encrypt);
995 if (err)
996 break;
997 list_del(&c->entry);
998 if (unlikely(c != &rctx->chunk))
999 kfree(c);
1000 }
1001 if (!err) {
1002 hv_ret = wait_for_tail(qp);
1003 if (hv_ret != HV_EOK)
1004 err = -EINVAL;
1005 }
1006
1007 spin_unlock_irqrestore(&qp->lock, flags);
1008
e27303b2 1009out:
0a625fd2
DM		 1010 put_cpu();
1011
0a625fd2
DM		 1012 n2_chunk_complete(req, NULL);
1013 return err;
1014}
1015
1016static int n2_encrypt_ecb(struct ablkcipher_request *req)
1017{
1018 return n2_do_ecb(req, true);
1019}
1020
1021static int n2_decrypt_ecb(struct ablkcipher_request *req)
1022{
1023 return n2_do_ecb(req, false);
1024}
1025
1026static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
1027{
1028 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
1029 struct crypto_tfm *tfm = req->base.tfm;
1030 unsigned long flags, hv_ret, iv_paddr;
1031 int err = n2_compute_chunks(req);
1032 struct n2_crypto_chunk *c, *tmp;
1033 struct spu_queue *qp;
1034 void *final_iv_addr;
1035
1036 final_iv_addr = NULL;
1037
1038 if (err)
1039 return err;
1040
1041 qp = cpu_to_cwq[get_cpu()];
1042 err = -ENODEV;
1043 if (!qp)
1044 goto out;
1045
1046 spin_lock_irqsave(&qp->lock, flags);
1047
1048 if (encrypt) {
1049 iv_paddr = __pa(rctx->walk.iv);
1050 list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
1051 entry) {
1052 c->iv_paddr = iv_paddr;
1053 err = __n2_crypt_chunk(tfm, c, qp, true);
1054 if (err)
1055 break;
1056 iv_paddr = c->dest_final - rctx->walk.blocksize;
1057 list_del(&c->entry);
1058 if (unlikely(c != &rctx->chunk))
1059 kfree(c);
1060 }
1061 final_iv_addr = __va(iv_paddr);
1062 } else {
1063 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1064 entry) {
1065 if (c == &rctx->chunk) {
1066 iv_paddr = __pa(rctx->walk.iv);
1067 } else {
1068 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1069 tmp->arr[tmp->arr_len-1].src_len -
1070 rctx->walk.blocksize);
1071 }
1072 if (!final_iv_addr) {
1073 unsigned long pa;
1074
1075 pa = (c->arr[c->arr_len-1].src_paddr +
1076 c->arr[c->arr_len-1].src_len -
1077 rctx->walk.blocksize);
1078 final_iv_addr = rctx->temp_iv;
1079 memcpy(rctx->temp_iv, __va(pa),
1080 rctx->walk.blocksize);
1081 }
1082 c->iv_paddr = iv_paddr;
1083 err = __n2_crypt_chunk(tfm, c, qp, false);
1084 if (err)
1085 break;
1086 list_del(&c->entry);
1087 if (unlikely(c != &rctx->chunk))
1088 kfree(c);
1089 }
1090 }
1091 if (!err) {
1092 hv_ret = wait_for_tail(qp);
1093 if (hv_ret != HV_EOK)
1094 err = -EINVAL;
1095 }
1096
1097 spin_unlock_irqrestore(&qp->lock, flags);
1098
e27303b2 1099out:
0a625fd2
DM		 1100 put_cpu();
1101
0a625fd2
DM		 1102 n2_chunk_complete(req, err ? NULL : final_iv_addr);
1103 return err;
1104}
1105
1106static int n2_encrypt_chaining(struct ablkcipher_request *req)
1107{
1108 return n2_do_chaining(req, true);
1109}
1110
1111static int n2_decrypt_chaining(struct ablkcipher_request *req)
1112{
1113 return n2_do_chaining(req, false);
1114}
1115
1116struct n2_cipher_tmpl {
1117 const char *name;
1118 const char *drv_name;
1119 u8 block_size;
1120 u8 enc_type;
1121 struct ablkcipher_alg ablkcipher;
1122};
1123
1124static const struct n2_cipher_tmpl cipher_tmpls[] = {
1125 /* ARC4: only ECB is supported (chaining bits ignored) */
1126 { .name = "ecb(arc4)",
1127 .drv_name = "ecb-arc4",
1128 .block_size = 1,
1129 .enc_type = (ENC_TYPE_ALG_RC4_STREAM |
1130 ENC_TYPE_CHAINING_ECB),
1131 .ablkcipher = {
1132 .min_keysize = 1,
1133 .max_keysize = 256,
1134 .setkey = n2_arc4_setkey,
1135 .encrypt = n2_encrypt_ecb,
1136 .decrypt = n2_decrypt_ecb,
1137 },
1138 },
1139
1140 /* DES: ECB CBC and CFB are supported */
1141 { .name = "ecb(des)",
1142 .drv_name = "ecb-des",
1143 .block_size = DES_BLOCK_SIZE,
1144 .enc_type = (ENC_TYPE_ALG_DES |
1145 ENC_TYPE_CHAINING_ECB),
1146 .ablkcipher = {
1147 .min_keysize = DES_KEY_SIZE,
1148 .max_keysize = DES_KEY_SIZE,
1149 .setkey = n2_des_setkey,
1150 .encrypt = n2_encrypt_ecb,
1151 .decrypt = n2_decrypt_ecb,
1152 },
1153 },
1154 { .name = "cbc(des)",
1155 .drv_name = "cbc-des",
1156 .block_size = DES_BLOCK_SIZE,
1157 .enc_type = (ENC_TYPE_ALG_DES |
1158 ENC_TYPE_CHAINING_CBC),
1159 .ablkcipher = {
1160 .ivsize = DES_BLOCK_SIZE,
1161 .min_keysize = DES_KEY_SIZE,
1162 .max_keysize = DES_KEY_SIZE,
1163 .setkey = n2_des_setkey,
1164 .encrypt = n2_encrypt_chaining,
1165 .decrypt = n2_decrypt_chaining,
1166 },
1167 },
1168 { .name = "cfb(des)",
1169 .drv_name = "cfb-des",
1170 .block_size = DES_BLOCK_SIZE,
1171 .enc_type = (ENC_TYPE_ALG_DES |
1172 ENC_TYPE_CHAINING_CFB),
1173 .ablkcipher = {
1174 .min_keysize = DES_KEY_SIZE,
1175 .max_keysize = DES_KEY_SIZE,
1176 .setkey = n2_des_setkey,
1177 .encrypt = n2_encrypt_chaining,
1178 .decrypt = n2_decrypt_chaining,
1179 },
1180 },
1181
1182 /* 3DES: ECB CBC and CFB are supported */
1183 { .name = "ecb(des3_ede)",
1184 .drv_name = "ecb-3des",
1185 .block_size = DES_BLOCK_SIZE,
1186 .enc_type = (ENC_TYPE_ALG_3DES |
1187 ENC_TYPE_CHAINING_ECB),
1188 .ablkcipher = {
1189 .min_keysize = 3 * DES_KEY_SIZE,
1190 .max_keysize = 3 * DES_KEY_SIZE,
1191 .setkey = n2_3des_setkey,
1192 .encrypt = n2_encrypt_ecb,
1193 .decrypt = n2_decrypt_ecb,
1194 },
1195 },
1196 { .name = "cbc(des3_ede)",
1197 .drv_name = "cbc-3des",
1198 .block_size = DES_BLOCK_SIZE,
1199 .enc_type = (ENC_TYPE_ALG_3DES |
1200 ENC_TYPE_CHAINING_CBC),
1201 .ablkcipher = {
1202 .ivsize = DES_BLOCK_SIZE,
1203 .min_keysize = 3 * DES_KEY_SIZE,
1204 .max_keysize = 3 * DES_KEY_SIZE,
1205 .setkey = n2_3des_setkey,
1206 .encrypt = n2_encrypt_chaining,
1207 .decrypt = n2_decrypt_chaining,
1208 },
1209 },
1210 { .name = "cfb(des3_ede)",
1211 .drv_name = "cfb-3des",
1212 .block_size = DES_BLOCK_SIZE,
1213 .enc_type = (ENC_TYPE_ALG_3DES |
1214 ENC_TYPE_CHAINING_CFB),
1215 .ablkcipher = {
1216 .min_keysize = 3 * DES_KEY_SIZE,
1217 .max_keysize = 3 * DES_KEY_SIZE,
1218 .setkey = n2_3des_setkey,
1219 .encrypt = n2_encrypt_chaining,
1220 .decrypt = n2_decrypt_chaining,
1221 },
1222 },
1223 /* AES: ECB CBC and CTR are supported */
1224 { .name = "ecb(aes)",
1225 .drv_name = "ecb-aes",
1226 .block_size = AES_BLOCK_SIZE,
1227 .enc_type = (ENC_TYPE_ALG_AES128 |
1228 ENC_TYPE_CHAINING_ECB),
1229 .ablkcipher = {
1230 .min_keysize = AES_MIN_KEY_SIZE,
1231 .max_keysize = AES_MAX_KEY_SIZE,
1232 .setkey = n2_aes_setkey,
1233 .encrypt = n2_encrypt_ecb,
1234 .decrypt = n2_decrypt_ecb,
1235 },
1236 },
1237 { .name = "cbc(aes)",
1238 .drv_name = "cbc-aes",
1239 .block_size = AES_BLOCK_SIZE,
1240 .enc_type = (ENC_TYPE_ALG_AES128 |
1241 ENC_TYPE_CHAINING_CBC),
1242 .ablkcipher = {
1243 .ivsize = AES_BLOCK_SIZE,
1244 .min_keysize = AES_MIN_KEY_SIZE,
1245 .max_keysize = AES_MAX_KEY_SIZE,
1246 .setkey = n2_aes_setkey,
1247 .encrypt = n2_encrypt_chaining,
1248 .decrypt = n2_decrypt_chaining,
1249 },
1250 },
1251 { .name = "ctr(aes)",
1252 .drv_name = "ctr-aes",
1253 .block_size = AES_BLOCK_SIZE,
1254 .enc_type = (ENC_TYPE_ALG_AES128 |
1255 ENC_TYPE_CHAINING_COUNTER),
1256 .ablkcipher = {
1257 .ivsize = AES_BLOCK_SIZE,
1258 .min_keysize = AES_MIN_KEY_SIZE,
1259 .max_keysize = AES_MAX_KEY_SIZE,
1260 .setkey = n2_aes_setkey,
1261 .encrypt = n2_encrypt_chaining,
1262 .decrypt = n2_encrypt_chaining,
1263 },
1264 },
1265
1266};
1267#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1268
1269static LIST_HEAD(cipher_algs);
1270
1271struct n2_hash_tmpl {
1272 const char *name;
3a2c0346
DM		 1273 const char *hash_zero;
1274 const u32 *hash_init;
1275 u8 hw_op_hashsz;
0a625fd2
DM		 1276 u8 digest_size;
1277 u8 block_size;
3a2c0346 1278 u8 auth_type;
dc4ccfd1 1279 u8 hmac_type;
3a2c0346
DM		 1280};
1281
1282static const char md5_zero[MD5_DIGEST_SIZE] = {
1283 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
1284 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
0a625fd2 1285};
3a2c0346
DM		 1286static const u32 md5_init[MD5_HASH_WORDS] = {
1287 cpu_to_le32(0x67452301),
1288 cpu_to_le32(0xefcdab89),
1289 cpu_to_le32(0x98badcfe),
1290 cpu_to_le32(0x10325476),
1291};
1292static const char sha1_zero[SHA1_DIGEST_SIZE] = {
1293 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
1294 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
1295 0x07, 0x09
1296};
1297static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
1298 SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
1299};
1300static const char sha256_zero[SHA256_DIGEST_SIZE] = {
1301 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
1302 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
1303 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
1304 0x1b, 0x78, 0x52, 0xb8, 0x55
1305};
1306static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
1307 SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
1308 SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
1309};
1310static const char sha224_zero[SHA224_DIGEST_SIZE] = {
1311 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
1312 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
1313 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
1314 0x2f
1315};
1316static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
1317 SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
1318 SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
1319};
1320
0a625fd2
DM		 1321static const struct n2_hash_tmpl hash_tmpls[] = {
1322 { .name = "md5",
3a2c0346
DM		 1323 .hash_zero = md5_zero,
1324 .hash_init = md5_init,
1325 .auth_type = AUTH_TYPE_MD5,
dc4ccfd1 1326 .hmac_type = AUTH_TYPE_HMAC_MD5,
3a2c0346 1327 .hw_op_hashsz = MD5_DIGEST_SIZE,
0a625fd2
DM		 1328 .digest_size = MD5_DIGEST_SIZE,
1329 .block_size = MD5_HMAC_BLOCK_SIZE },
1330 { .name = "sha1",
3a2c0346
DM		 1331 .hash_zero = sha1_zero,
1332 .hash_init = sha1_init,
1333 .auth_type = AUTH_TYPE_SHA1,
dc4ccfd1 1334 .hmac_type = AUTH_TYPE_HMAC_SHA1,
3a2c0346 1335 .hw_op_hashsz = SHA1_DIGEST_SIZE,
0a625fd2
DM		 1336 .digest_size = SHA1_DIGEST_SIZE,
1337 .block_size = SHA1_BLOCK_SIZE },
1338 { .name = "sha256",
3a2c0346
DM		 1339 .hash_zero = sha256_zero,
1340 .hash_init = sha256_init,
1341 .auth_type = AUTH_TYPE_SHA256,
dc4ccfd1 1342 .hmac_type = AUTH_TYPE_HMAC_SHA256,
3a2c0346 1343 .hw_op_hashsz = SHA256_DIGEST_SIZE,
0a625fd2
DM		 1344 .digest_size = SHA256_DIGEST_SIZE,
1345 .block_size = SHA256_BLOCK_SIZE },
1346 { .name = "sha224",
3a2c0346
DM		 1347 .hash_zero = sha224_zero,
1348 .hash_init = sha224_init,
1349 .auth_type = AUTH_TYPE_SHA256,
dc4ccfd1 1350 .hmac_type = AUTH_TYPE_RESERVED,
3a2c0346 1351 .hw_op_hashsz = SHA256_DIGEST_SIZE,
0a625fd2
DM		 1352 .digest_size = SHA224_DIGEST_SIZE,
1353 .block_size = SHA224_BLOCK_SIZE },
1354};
1355#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1356
0a625fd2 1357static LIST_HEAD(ahash_algs);
dc4ccfd1 1358static LIST_HEAD(hmac_algs);
0a625fd2
DM		 1359
1360static int algs_registered;
1361
1362static void __n2_unregister_algs(void)
1363{
1364 struct n2_cipher_alg *cipher, *cipher_tmp;
1365 struct n2_ahash_alg *alg, *alg_tmp;
dc4ccfd1 1366 struct n2_hmac_alg *hmac, *hmac_tmp;
0a625fd2
DM		 1367
1368 list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1369 crypto_unregister_alg(&cipher->alg);
1370 list_del(&cipher->entry);
1371 kfree(cipher);
1372 }
dc4ccfd1
DM		 1373 list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1374 crypto_unregister_ahash(&hmac->derived.alg);
1375 list_del(&hmac->derived.entry);
1376 kfree(hmac);
1377 }
0a625fd2
DM		 1378 list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1379 crypto_unregister_ahash(&alg->alg);
1380 list_del(&alg->entry);
1381 kfree(alg);
1382 }
1383}
1384
1385static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1386{
1387 tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1388 return 0;
1389}
1390
49cfe4db 1391static int __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
0a625fd2
DM		 1392{
1393 struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1394 struct crypto_alg *alg;
1395 int err;
1396
1397 if (!p)
1398 return -ENOMEM;
1399
1400 alg = &p->alg;
1401
1402 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1403 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1404 alg->cra_priority = N2_CRA_PRIORITY;
d912bb76
NM		 1405 alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1406 CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
0a625fd2
DM		 1407 alg->cra_blocksize = tmpl->block_size;
1408 p->enc_type = tmpl->enc_type;
1409 alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1410 alg->cra_type = &crypto_ablkcipher_type;
1411 alg->cra_u.ablkcipher = tmpl->ablkcipher;
1412 alg->cra_init = n2_cipher_cra_init;
1413 alg->cra_module = THIS_MODULE;
1414
1415 list_add(&p->entry, &cipher_algs);
1416 err = crypto_register_alg(alg);
1417 if (err) {
38511108 1418 pr_err("%s alg registration failed\n", alg->cra_name);
0a625fd2
DM		 1419 list_del(&p->entry);
1420 kfree(p);
38511108
DM		 1421 } else {
1422 pr_info("%s alg registered\n", alg->cra_name);
0a625fd2
DM		 1423 }
1424 return err;
1425}
1426
49cfe4db 1427static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
dc4ccfd1
DM		 1428{
1429 struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1430 struct ahash_alg *ahash;
1431 struct crypto_alg *base;
1432 int err;
1433
1434 if (!p)
1435 return -ENOMEM;
1436
1437 p->child_alg = n2ahash->alg.halg.base.cra_name;
1438 memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1439 INIT_LIST_HEAD(&p->derived.entry);
1440
1441 ahash = &p->derived.alg;
1442 ahash->digest = n2_hmac_async_digest;
1443 ahash->setkey = n2_hmac_async_setkey;
1444
1445 base = &ahash->halg.base;
1446 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
1447 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
1448
1449 base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1450 base->cra_init = n2_hmac_cra_init;
1451 base->cra_exit = n2_hmac_cra_exit;
1452
1453 list_add(&p->derived.entry, &hmac_algs);
1454 err = crypto_register_ahash(ahash);
1455 if (err) {
1456 pr_err("%s alg registration failed\n", base->cra_name);
1457 list_del(&p->derived.entry);
1458 kfree(p);
1459 } else {
1460 pr_info("%s alg registered\n", base->cra_name);
1461 }
1462 return err;
1463}
1464
49cfe4db 1465static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
0a625fd2
DM		 1466{
1467 struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1468 struct hash_alg_common *halg;
1469 struct crypto_alg *base;
1470 struct ahash_alg *ahash;
1471 int err;
1472
1473 if (!p)
1474 return -ENOMEM;
1475
3a2c0346
DM		 1476 p->hash_zero = tmpl->hash_zero;
1477 p->hash_init = tmpl->hash_init;
1478 p->auth_type = tmpl->auth_type;
dc4ccfd1 1479 p->hmac_type = tmpl->hmac_type;
3a2c0346
DM		 1480 p->hw_op_hashsz = tmpl->hw_op_hashsz;
1481 p->digest_size = tmpl->digest_size;
1482
0a625fd2
DM		 1483 ahash = &p->alg;
1484 ahash->init = n2_hash_async_init;
1485 ahash->update = n2_hash_async_update;
1486 ahash->final = n2_hash_async_final;
1487 ahash->finup = n2_hash_async_finup;
3a2c0346 1488 ahash->digest = n2_hash_async_digest;
0a625fd2
DM		 1489
1490 halg = &ahash->halg;
1491 halg->digestsize = tmpl->digest_size;
1492
1493 base = &halg->base;
1494 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1495 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1496 base->cra_priority = N2_CRA_PRIORITY;
d912bb76
NM		 1497 base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
1498 CRYPTO_ALG_KERN_DRIVER_ONLY |
1499 CRYPTO_ALG_NEED_FALLBACK;
0a625fd2
DM		 1500 base->cra_blocksize = tmpl->block_size;
1501 base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1502 base->cra_module = THIS_MODULE;
1503 base->cra_init = n2_hash_cra_init;
1504 base->cra_exit = n2_hash_cra_exit;
1505
1506 list_add(&p->entry, &ahash_algs);
1507 err = crypto_register_ahash(ahash);
1508 if (err) {
38511108 1509 pr_err("%s alg registration failed\n", base->cra_name);
0a625fd2
DM		 1510 list_del(&p->entry);
1511 kfree(p);
38511108
DM		 1512 } else {
1513 pr_info("%s alg registered\n", base->cra_name);
0a625fd2 1514 }
dc4ccfd1
DM		 1515 if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1516 err = __n2_register_one_hmac(p);
0a625fd2
DM		 1517 return err;
1518}
1519
49cfe4db 1520static int n2_register_algs(void)
0a625fd2
DM		 1521{
1522 int i, err = 0;
1523
1524 mutex_lock(&spu_lock);
1525 if (algs_registered++)
1526 goto out;
1527
1528 for (i = 0; i < NUM_HASH_TMPLS; i++) {
1529 err = __n2_register_one_ahash(&hash_tmpls[i]);
1530 if (err) {
1531 __n2_unregister_algs();
1532 goto out;
1533 }
1534 }
1535 for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1536 err = __n2_register_one_cipher(&cipher_tmpls[i]);
1537 if (err) {
1538 __n2_unregister_algs();
1539 goto out;
1540 }
1541 }
1542
1543out:
1544 mutex_unlock(&spu_lock);
1545 return err;
1546}
1547
49cfe4db 1548static void n2_unregister_algs(void)
0a625fd2
DM		 1549{
1550 mutex_lock(&spu_lock);
1551 if (!--algs_registered)
1552 __n2_unregister_algs();
1553 mutex_unlock(&spu_lock);
1554}
1555
1556/* To map CWQ queues to interrupt sources, the hypervisor API provides
1557 * a devino. This isn't very useful to us because all of the
2dc11581 1558 * interrupts listed in the device_node have been translated to
0a625fd2
DM		 1559 * Linux virtual IRQ cookie numbers.
1560 *
1561 * So we have to back-translate, going through the 'intr' and 'ino'
1562 * property tables of the n2cp MDESC node, matching it with the OF
1563 * 'interrupts' property entries, in order to to figure out which
1564 * devino goes to which already-translated IRQ.
1565 */
2dc11581 1566static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
0a625fd2
DM		 1567 unsigned long dev_ino)
1568{
1569 const unsigned int *dev_intrs;
1570 unsigned int intr;
1571 int i;
1572
1573 for (i = 0; i < ip->num_intrs; i++) {
1574 if (ip->ino_table[i].ino == dev_ino)
1575 break;
1576 }
1577 if (i == ip->num_intrs)
1578 return -ENODEV;
1579
1580 intr = ip->ino_table[i].intr;
1581
ff6c7341 1582 dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
0a625fd2
DM		 1583 if (!dev_intrs)
1584 return -ENODEV;
1585
19e4875f 1586 for (i = 0; i < dev->archdata.num_irqs; i++) {
0a625fd2
DM		 1587 if (dev_intrs[i] == intr)
1588 return i;
1589 }
1590
1591 return -ENODEV;
1592}
1593
2dc11581 1594static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
0a625fd2
DM		 1595 const char *irq_name, struct spu_queue *p,
1596 irq_handler_t handler)
1597{
1598 unsigned long herr;
1599 int index;
1600
1601 herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1602 if (herr)
1603 return -EINVAL;
1604
1605 index = find_devino_index(dev, ip, p->devino);
1606 if (index < 0)
1607 return index;
1608
19e4875f 1609 p->irq = dev->archdata.irqs[index];
0a625fd2
DM		 1610
1611 sprintf(p->irq_name, "%s-%d", irq_name, index);
1612
9751bfd1 1613 return request_irq(p->irq, handler, 0, p->irq_name, p);
0a625fd2
DM		 1614}
1615
1616static struct kmem_cache *queue_cache[2];
1617
1618static void *new_queue(unsigned long q_type)
1619{
1620 return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1621}
1622
1623static void free_queue(void *p, unsigned long q_type)
1624{
1625 return kmem_cache_free(queue_cache[q_type - 1], p);
1626}
1627
1628static int queue_cache_init(void)
1629{
1630 if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1631 queue_cache[HV_NCS_QTYPE_MAU - 1] =
527b9525 1632 kmem_cache_create("mau_queue",
0a625fd2
DM		 1633 (MAU_NUM_ENTRIES *
1634 MAU_ENTRY_SIZE),
1635 MAU_ENTRY_SIZE, 0, NULL);
1636 if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1637 return -ENOMEM;
1638
1639 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1640 queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1641 kmem_cache_create("cwq_queue",
1642 (CWQ_NUM_ENTRIES *
1643 CWQ_ENTRY_SIZE),
1644 CWQ_ENTRY_SIZE, 0, NULL);
1645 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1646 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1647 return -ENOMEM;
1648 }
1649 return 0;
1650}
1651
1652static void queue_cache_destroy(void)
1653{
1654 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1655 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1656}
1657
1658static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1659{
1660 cpumask_var_t old_allowed;
1661 unsigned long hv_ret;
1662
1663 if (cpumask_empty(&p->sharing))
1664 return -EINVAL;
1665
1666 if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1667 return -ENOMEM;
1668
1669 cpumask_copy(old_allowed, &current->cpus_allowed);
1670
1671 set_cpus_allowed_ptr(current, &p->sharing);
1672
1673 hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1674 CWQ_NUM_ENTRIES, &p->qhandle);
1675 if (!hv_ret)
1676 sun4v_ncs_sethead_marker(p->qhandle, 0);
1677
1678 set_cpus_allowed_ptr(current, old_allowed);
1679
1680 free_cpumask_var(old_allowed);
1681
1682 return (hv_ret ? -EINVAL : 0);
1683}
1684
1685static int spu_queue_setup(struct spu_queue *p)
1686{
1687 int err;
1688
1689 p->q = new_queue(p->q_type);
1690 if (!p->q)
1691 return -ENOMEM;
1692
1693 err = spu_queue_register(p, p->q_type);
1694 if (err) {
1695 free_queue(p->q, p->q_type);
1696 p->q = NULL;
1697 }
1698
1699 return err;
1700}
1701
1702static void spu_queue_destroy(struct spu_queue *p)
1703{
1704 unsigned long hv_ret;
1705
1706 if (!p->q)
1707 return;
1708
1709 hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1710
1711 if (!hv_ret)
1712 free_queue(p->q, p->q_type);
1713}
1714
1715static void spu_list_destroy(struct list_head *list)
1716{
1717 struct spu_queue *p, *n;
1718
1719 list_for_each_entry_safe(p, n, list, list) {
1720 int i;
1721
1722 for (i = 0; i < NR_CPUS; i++) {
1723 if (cpu_to_cwq[i] == p)
1724 cpu_to_cwq[i] = NULL;
1725 }
1726
1727 if (p->irq) {
1728 free_irq(p->irq, p);
1729 p->irq = 0;
1730 }
1731 spu_queue_destroy(p);
1732 list_del(&p->list);
1733 kfree(p);
1734 }
1735}
1736
1737/* Walk the backward arcs of a CWQ 'exec-unit' node,
1738 * gathering cpu membership information.
1739 */
1740static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
2dc11581 1741 struct platform_device *dev,
0a625fd2
DM		 1742 u64 node, struct spu_queue *p,
1743 struct spu_queue **table)
1744{
1745 u64 arc;
1746
1747 mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1748 u64 tgt = mdesc_arc_target(mdesc, arc);
1749 const char *name = mdesc_node_name(mdesc, tgt);
1750 const u64 *id;
1751
1752 if (strcmp(name, "cpu"))
1753 continue;
1754 id = mdesc_get_property(mdesc, tgt, "id", NULL);
1755 if (table[*id] != NULL) {
1756 dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
ff6c7341 1757 dev->dev.of_node->full_name);
0a625fd2
DM		 1758 return -EINVAL;
1759 }
1760 cpu_set(*id, p->sharing);
1761 table[*id] = p;
1762 }
1763 return 0;
1764}
1765
1766/* Process an 'exec-unit' MDESC node of type 'cwq'. */
1767static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
2dc11581 1768 struct platform_device *dev, struct mdesc_handle *mdesc,
0a625fd2
DM		 1769 u64 node, const char *iname, unsigned long q_type,
1770 irq_handler_t handler, struct spu_queue **table)
1771{
1772 struct spu_queue *p;
1773 int err;
1774
1775 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1776 if (!p) {
1777 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
ff6c7341 1778 dev->dev.of_node->full_name);
0a625fd2
DM		 1779 return -ENOMEM;
1780 }
1781
1782 cpus_clear(p->sharing);
1783 spin_lock_init(&p->lock);
1784 p->q_type = q_type;
1785 INIT_LIST_HEAD(&p->jobs);
1786 list_add(&p->list, list);
1787
1788 err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1789 if (err)
1790 return err;
1791
1792 err = spu_queue_setup(p);
1793 if (err)
1794 return err;
1795
1796 return spu_map_ino(dev, ip, iname, p, handler);
1797}
1798
2dc11581 1799static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
0a625fd2
DM		 1800 struct spu_mdesc_info *ip, struct list_head *list,
1801 const char *exec_name, unsigned long q_type,
1802 irq_handler_t handler, struct spu_queue **table)
1803{
1804 int err = 0;
1805 u64 node;
1806
1807 mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1808 const char *type;
1809
1810 type = mdesc_get_property(mdesc, node, "type", NULL);
1811 if (!type || strcmp(type, exec_name))
1812 continue;
1813
1814 err = handle_exec_unit(ip, list, dev, mdesc, node,
1815 exec_name, q_type, handler, table);
1816 if (err) {
1817 spu_list_destroy(list);
1818 break;
1819 }
1820 }
1821
1822 return err;
1823}
1824
49cfe4db
GKH		 1825static int get_irq_props(struct mdesc_handle *mdesc, u64 node,
1826 struct spu_mdesc_info *ip)
0a625fd2 1827{
eb7caf35
DM		 1828 const u64 *ino;
1829 int ino_len;
0a625fd2
DM		 1830 int i;
1831
0a625fd2 1832 ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
eb7caf35
DM		 1833 if (!ino) {
1834 printk("NO 'ino'\n");
0a625fd2 1835 return -ENODEV;
eb7caf35 1836 }
0a625fd2 1837
eb7caf35 1838 ip->num_intrs = ino_len / sizeof(u64);
0a625fd2
DM		 1839 ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1840 ip->num_intrs),
1841 GFP_KERNEL);
1842 if (!ip->ino_table)
1843 return -ENOMEM;
1844
1845 for (i = 0; i < ip->num_intrs; i++) {
1846 struct ino_blob *b = &ip->ino_table[i];
eb7caf35 1847 b->intr = i + 1;
0a625fd2
DM		 1848 b->ino = ino[i];
1849 }
1850
1851 return 0;
1852}
1853
49cfe4db
GKH		 1854static int grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1855 struct platform_device *dev,
1856 struct spu_mdesc_info *ip,
1857 const char *node_name)
0a625fd2
DM		 1858{
1859 const unsigned int *reg;
1860 u64 node;
1861
ff6c7341 1862 reg = of_get_property(dev->dev.of_node, "reg", NULL);
0a625fd2
DM		 1863 if (!reg)
1864 return -ENODEV;
1865
1866 mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1867 const char *name;
1868 const u64 *chdl;
1869
1870 name = mdesc_get_property(mdesc, node, "name", NULL);
1871 if (!name || strcmp(name, node_name))
1872 continue;
1873 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1874 if (!chdl || (*chdl != *reg))
1875 continue;
1876 ip->cfg_handle = *chdl;
1877 return get_irq_props(mdesc, node, ip);
1878 }
1879
1880 return -ENODEV;
1881}
1882
1883static unsigned long n2_spu_hvapi_major;
1884static unsigned long n2_spu_hvapi_minor;
1885
49cfe4db 1886static int n2_spu_hvapi_register(void)
0a625fd2
DM		 1887{
1888 int err;
1889
1890 n2_spu_hvapi_major = 2;
1891 n2_spu_hvapi_minor = 0;
1892
1893 err = sun4v_hvapi_register(HV_GRP_NCS,
1894 n2_spu_hvapi_major,
1895 &n2_spu_hvapi_minor);
1896
1897 if (!err)
1898 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1899 n2_spu_hvapi_major,
1900 n2_spu_hvapi_minor);
1901
1902 return err;
1903}
1904
1905static void n2_spu_hvapi_unregister(void)
1906{
1907 sun4v_hvapi_unregister(HV_GRP_NCS);
1908}
1909
1910static int global_ref;
1911
49cfe4db 1912static int grab_global_resources(void)
0a625fd2
DM		 1913{
1914 int err = 0;
1915
1916 mutex_lock(&spu_lock);
1917
1918 if (global_ref++)
1919 goto out;
1920
1921 err = n2_spu_hvapi_register();
1922 if (err)
1923 goto out;
1924
1925 err = queue_cache_init();
1926 if (err)
1927 goto out_hvapi_release;
1928
1929 err = -ENOMEM;
1930 cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1931 GFP_KERNEL);
1932 if (!cpu_to_cwq)
1933 goto out_queue_cache_destroy;
1934
1935 cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1936 GFP_KERNEL);
1937 if (!cpu_to_mau)
1938 goto out_free_cwq_table;
1939
1940 err = 0;
1941
1942out:
1943 if (err)
1944 global_ref--;
1945 mutex_unlock(&spu_lock);
1946 return err;
1947
1948out_free_cwq_table:
1949 kfree(cpu_to_cwq);
1950 cpu_to_cwq = NULL;
1951
1952out_queue_cache_destroy:
1953 queue_cache_destroy();
1954
1955out_hvapi_release:
1956 n2_spu_hvapi_unregister();
1957 goto out;
1958}
1959
1960static void release_global_resources(void)
1961{
1962 mutex_lock(&spu_lock);
1963 if (!--global_ref) {
1964 kfree(cpu_to_cwq);
1965 cpu_to_cwq = NULL;
1966
1967 kfree(cpu_to_mau);
1968 cpu_to_mau = NULL;
1969
1970 queue_cache_destroy();
1971 n2_spu_hvapi_unregister();
1972 }
1973 mutex_unlock(&spu_lock);
1974}
1975
49cfe4db 1976static struct n2_crypto *alloc_n2cp(void)
0a625fd2
DM		 1977{
1978 struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1979
1980 if (np)
1981 INIT_LIST_HEAD(&np->cwq_list);
1982
1983 return np;
1984}
1985
1986static void free_n2cp(struct n2_crypto *np)
1987{
1988 if (np->cwq_info.ino_table) {
1989 kfree(np->cwq_info.ino_table);
1990 np->cwq_info.ino_table = NULL;
1991 }
1992
1993 kfree(np);
1994}
1995
49cfe4db 1996static void n2_spu_driver_version(void)
0a625fd2
DM		 1997{
1998 static int n2_spu_version_printed;
1999
2000 if (n2_spu_version_printed++ == 0)
2001 pr_info("%s", version);
2002}
2003
49cfe4db 2004static int n2_crypto_probe(struct platform_device *dev)
0a625fd2
DM		 2005{
2006 struct mdesc_handle *mdesc;
2007 const char *full_name;
2008 struct n2_crypto *np;
2009 int err;
2010
2011 n2_spu_driver_version();
2012
ff6c7341 2013 full_name = dev->dev.of_node->full_name;
0a625fd2
DM		 2014 pr_info("Found N2CP at %s\n", full_name);
2015
2016 np = alloc_n2cp();
2017 if (!np) {
2018 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
2019 full_name);
2020 return -ENOMEM;
2021 }
2022
2023 err = grab_global_resources();
2024 if (err) {
2025 dev_err(&dev->dev, "%s: Unable to grab "
2026 "global resources.\n", full_name);
2027 goto out_free_n2cp;
2028 }
2029
2030 mdesc = mdesc_grab();
2031
2032 if (!mdesc) {
2033 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2034 full_name);
2035 err = -ENODEV;
2036 goto out_free_global;
2037 }
2038 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2039 if (err) {
2040 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2041 full_name);
2042 mdesc_release(mdesc);
2043 goto out_free_global;
2044 }
2045
2046 err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
2047 "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
2048 cpu_to_cwq);
2049 mdesc_release(mdesc);
2050
2051 if (err) {
2052 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
2053 full_name);
2054 goto out_free_global;
2055 }
2056
2057 err = n2_register_algs();
2058 if (err) {
2059 dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
2060 full_name);
2061 goto out_free_spu_list;
2062 }
2063
2064 dev_set_drvdata(&dev->dev, np);
2065
2066 return 0;
2067
2068out_free_spu_list:
2069 spu_list_destroy(&np->cwq_list);
2070
2071out_free_global:
2072 release_global_resources();
2073
2074out_free_n2cp:
2075 free_n2cp(np);
2076
2077 return err;
2078}
2079
49cfe4db 2080static int n2_crypto_remove(struct platform_device *dev)
0a625fd2
DM		 2081{
2082 struct n2_crypto *np = dev_get_drvdata(&dev->dev);
2083
2084 n2_unregister_algs();
2085
2086 spu_list_destroy(&np->cwq_list);
2087
2088 release_global_resources();
2089
2090 free_n2cp(np);
2091
2092 return 0;
2093}
2094
49cfe4db 2095static struct n2_mau *alloc_ncp(void)
0a625fd2
DM		 2096{
2097 struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
2098
2099 if (mp)
2100 INIT_LIST_HEAD(&mp->mau_list);
2101
2102 return mp;
2103}
2104
2105static void free_ncp(struct n2_mau *mp)
2106{
2107 if (mp->mau_info.ino_table) {
2108 kfree(mp->mau_info.ino_table);
2109 mp->mau_info.ino_table = NULL;
2110 }
2111
2112 kfree(mp);
2113}
2114
49cfe4db 2115static int n2_mau_probe(struct platform_device *dev)
0a625fd2
DM		 2116{
2117 struct mdesc_handle *mdesc;
2118 const char *full_name;
2119 struct n2_mau *mp;
2120 int err;
2121
2122 n2_spu_driver_version();
2123
ff6c7341 2124 full_name = dev->dev.of_node->full_name;
0a625fd2
DM		 2125 pr_info("Found NCP at %s\n", full_name);
2126
2127 mp = alloc_ncp();
2128 if (!mp) {
2129 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
2130 full_name);
2131 return -ENOMEM;
2132 }
2133
2134 err = grab_global_resources();
2135 if (err) {
2136 dev_err(&dev->dev, "%s: Unable to grab "
2137 "global resources.\n", full_name);
2138 goto out_free_ncp;
2139 }
2140
2141 mdesc = mdesc_grab();
2142
2143 if (!mdesc) {
2144 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2145 full_name);
2146 err = -ENODEV;
2147 goto out_free_global;
2148 }
2149
2150 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2151 if (err) {
2152 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2153 full_name);
2154 mdesc_release(mdesc);
2155 goto out_free_global;
2156 }
2157
2158 err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
2159 "mau", HV_NCS_QTYPE_MAU, mau_intr,
2160 cpu_to_mau);
2161 mdesc_release(mdesc);
2162
2163 if (err) {
2164 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
2165 full_name);
2166 goto out_free_global;
2167 }
2168
2169 dev_set_drvdata(&dev->dev, mp);
2170
2171 return 0;
2172
2173out_free_global:
2174 release_global_resources();
2175
2176out_free_ncp:
2177 free_ncp(mp);
2178
2179 return err;
2180}
2181
49cfe4db 2182static int n2_mau_remove(struct platform_device *dev)
0a625fd2
DM		 2183{
2184 struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2185
2186 spu_list_destroy(&mp->mau_list);
2187
2188 release_global_resources();
2189
2190 free_ncp(mp);
2191
2192 return 0;
2193}
2194
2195static struct of_device_id n2_crypto_match[] = {
2196 {
2197 .name = "n2cp",
2198 .compatible = "SUNW,n2-cwq",
2199 },
2200 {
2201 .name = "n2cp",
2202 .compatible = "SUNW,vf-cwq",
2203 },
eb7caf35
DM		 2204 {
2205 .name = "n2cp",
2206 .compatible = "SUNW,kt-cwq",
2207 },
0a625fd2
DM		 2208 {},
2209};
2210
2211MODULE_DEVICE_TABLE(of, n2_crypto_match);
2212
4ebb24f7 2213static struct platform_driver n2_crypto_driver = {
ff6c7341
DM		 2214 .driver = {
2215 .name = "n2cp",
2216 .owner = THIS_MODULE,
2217 .of_match_table = n2_crypto_match,
2218 },
0a625fd2 2219 .probe = n2_crypto_probe,
49cfe4db 2220 .remove = n2_crypto_remove,
0a625fd2
DM		 2221};
2222
2223static struct of_device_id n2_mau_match[] = {
2224 {
2225 .name = "ncp",
2226 .compatible = "SUNW,n2-mau",
2227 },
2228 {
2229 .name = "ncp",
2230 .compatible = "SUNW,vf-mau",
2231 },
eb7caf35
DM		 2232 {
2233 .name = "ncp",
2234 .compatible = "SUNW,kt-mau",
2235 },
0a625fd2
DM		 2236 {},
2237};
2238
2239MODULE_DEVICE_TABLE(of, n2_mau_match);
2240
4ebb24f7 2241static struct platform_driver n2_mau_driver = {
ff6c7341
DM		 2242 .driver = {
2243 .name = "ncp",
2244 .owner = THIS_MODULE,
2245 .of_match_table = n2_mau_match,
2246 },
0a625fd2 2247 .probe = n2_mau_probe,
49cfe4db 2248 .remove = n2_mau_remove,
0a625fd2
DM		 2249};
2250
2251static int __init n2_init(void)
2252{
4ebb24f7 2253 int err = platform_driver_register(&n2_crypto_driver);
0a625fd2
DM		 2254
2255 if (!err) {
4ebb24f7 2256 err = platform_driver_register(&n2_mau_driver);
0a625fd2 2257 if (err)
4ebb24f7 2258 platform_driver_unregister(&n2_crypto_driver);
0a625fd2
DM		 2259 }
2260 return err;
2261}
2262
2263static void __exit n2_exit(void)
2264{
4ebb24f7
GL		 2265 platform_driver_unregister(&n2_mau_driver);
2266 platform_driver_unregister(&n2_crypto_driver);
0a625fd2
DM		 2267}
2268
2269module_init(n2_init);
2270module_exit(n2_exit);
Linux kernel - Deliverables
This page took 0.321299 seconds and 5 git commands to generate.