projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'omap_dss2/for-next'
[deliverable/linux.git] / drivers / crypto / ccp / ccp-dev-v5.c
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/kthread.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/interrupt.h>
19 #include <linux/compiler.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
25 {
26 struct ccp_device *ccp;
27 int start;
28
29 /* First look at the map for the queue */
30 if (cmd_q->lsb >= 0) {
31 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
32 LSB_SIZE,
33 0, count, 0);
34 if (start < LSB_SIZE) {
35 bitmap_set(cmd_q->lsbmap, start, count);
36 return start + cmd_q->lsb * LSB_SIZE;
37 }
38 }
39
40 /* No joy; try to get an entry from the shared blocks */
41 ccp = cmd_q->ccp;
42 for (;;) {
43 mutex_lock(&ccp->sb_mutex);
44
45 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
46 MAX_LSB_CNT * LSB_SIZE,
47 0,
48 count, 0);
49 if (start <= MAX_LSB_CNT * LSB_SIZE) {
50 bitmap_set(ccp->lsbmap, start, count);
51
52 mutex_unlock(&ccp->sb_mutex);
53 return start * LSB_ITEM_SIZE;
54 }
55
56 ccp->sb_avail = 0;
57
58 mutex_unlock(&ccp->sb_mutex);
59
60 /* Wait for KSB entries to become available */
61 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
62 return 0;
63 }
64 }
65
66 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
67 unsigned int count)
68 {
69 int lsbno = start / LSB_SIZE;
70
71 if (!start)
72 return;
73
74 if (cmd_q->lsb == lsbno) {
75 /* An entry from the private LSB */
76 bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
77 } else {
78 /* From the shared LSBs */
79 struct ccp_device *ccp = cmd_q->ccp;
80
81 mutex_lock(&ccp->sb_mutex);
82 bitmap_clear(ccp->lsbmap, start, count);
83 ccp->sb_avail = 1;
84 mutex_unlock(&ccp->sb_mutex);
85 wake_up_interruptible_all(&ccp->sb_queue);
86 }
87 }
88
89 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
90 union ccp_function {
91 struct {
92 u16 size:7;
93 u16 encrypt:1;
94 u16 mode:5;
95 u16 type:2;
96 } aes;
97 struct {
98 u16 size:7;
99 u16 encrypt:1;
100 u16 rsvd:5;
101 u16 type:2;
102 } aes_xts;
103 struct {
104 u16 rsvd1:10;
105 u16 type:4;
106 u16 rsvd2:1;
107 } sha;
108 struct {
109 u16 mode:3;
110 u16 size:12;
111 } rsa;
112 struct {
113 u16 byteswap:2;
114 u16 bitwise:3;
115 u16 reflect:2;
116 u16 rsvd:8;
117 } pt;
118 struct {
119 u16 rsvd:13;
120 } zlib;
121 struct {
122 u16 size:10;
123 u16 type:2;
124 u16 mode:3;
125 } ecc;
126 u16 raw;
127 };
128
129 #define CCP_AES_SIZE(p) ((p)->aes.size)
130 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
131 #define CCP_AES_MODE(p) ((p)->aes.mode)
132 #define CCP_AES_TYPE(p) ((p)->aes.type)
133 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
134 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
135 #define CCP_SHA_TYPE(p) ((p)->sha.type)
136 #define CCP_RSA_SIZE(p) ((p)->rsa.size)
137 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
138 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
139 #define CCP_ECC_MODE(p) ((p)->ecc.mode)
140 #define CCP_ECC_AFFINE(p) ((p)->ecc.one)
141
142 /* Word 0 */
143 #define CCP5_CMD_DW0(p) ((p)->dw0)
144 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
145 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
146 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
147 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
148 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
149 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
150 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
151
152 /* Word 1 */
153 #define CCP5_CMD_DW1(p) ((p)->length)
154 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
155
156 /* Word 2 */
157 #define CCP5_CMD_DW2(p) ((p)->src_lo)
158 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
159
160 /* Word 3 */
161 #define CCP5_CMD_DW3(p) ((p)->dw3)
162 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
163 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
164 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
165 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
166
167 /* Words 4/5 */
168 #define CCP5_CMD_DW4(p) ((p)->dw4)
169 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
170 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
171 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
172 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
173 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
174 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
175 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
176
177 /* Word 6/7 */
178 #define CCP5_CMD_DW6(p) ((p)->key_lo)
179 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
180 #define CCP5_CMD_DW7(p) ((p)->dw7)
181 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
182 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
183
184 static inline u32 low_address(unsigned long addr)
185 {
186 return (u64)addr & 0x0ffffffff;
187 }
188
189 static inline u32 high_address(unsigned long addr)
190 {
191 return ((u64)addr >> 32) & 0x00000ffff;
192 }
193
194 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
195 {
196 unsigned int head_idx, n;
197 u32 head_lo, queue_start;
198
199 queue_start = low_address(cmd_q->qdma_tail);
200 head_lo = ioread32(cmd_q->reg_head_lo);
201 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
202
203 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
204
205 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
206 }
207
208 static int ccp5_do_cmd(struct ccp5_desc *desc,
209 struct ccp_cmd_queue *cmd_q)
210 {
211 u32 *mP;
212 __le32 *dP;
213 u32 tail;
214 int i;
215 int ret = 0;
216
217 if (CCP5_CMD_SOC(desc)) {
218 CCP5_CMD_IOC(desc) = 1;
219 CCP5_CMD_SOC(desc) = 0;
220 }
221 mutex_lock(&cmd_q->q_mutex);
222
223 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
224 dP = (__le32 *) desc;
225 for (i = 0; i < 8; i++)
226 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
227
228 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
229
230 /* The data used by this command must be flushed to memory */
231 wmb();
232
233 /* Write the new tail address back to the queue register */
234 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
235 iowrite32(tail, cmd_q->reg_tail_lo);
236
237 /* Turn the queue back on using our cached control register */
238 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
239 mutex_unlock(&cmd_q->q_mutex);
240
241 if (CCP5_CMD_IOC(desc)) {
242 /* Wait for the job to complete */
243 ret = wait_event_interruptible(cmd_q->int_queue,
244 cmd_q->int_rcvd);
245 if (ret || cmd_q->cmd_error) {
246 /* A version 5 device doesn't use Job IDs... */
247 if (!ret)
248 ret = -EIO;
249 }
250 cmd_q->int_rcvd = 0;
251 }
252
253 return 0;
254 }
255
256 static int ccp5_perform_aes(struct ccp_op *op)
257 {
258 struct ccp5_desc desc;
259 union ccp_function function;
260 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
261
262 /* Zero out all the fields of the command desc */
263 memset(&desc, 0, Q_DESC_SIZE);
264
265 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
266
267 CCP5_CMD_SOC(&desc) = op->soc;
268 CCP5_CMD_IOC(&desc) = 1;
269 CCP5_CMD_INIT(&desc) = op->init;
270 CCP5_CMD_EOM(&desc) = op->eom;
271 CCP5_CMD_PROT(&desc) = 0;
272
273 function.raw = 0;
274 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
275 CCP_AES_MODE(&function) = op->u.aes.mode;
276 CCP_AES_TYPE(&function) = op->u.aes.type;
277 if (op->u.aes.mode == CCP_AES_MODE_CFB)
278 CCP_AES_SIZE(&function) = 0x7f;
279
280 CCP5_CMD_FUNCTION(&desc) = function.raw;
281
282 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
283
284 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
285 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
286 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
287
288 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
289 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
290 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
291
292 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
293 CCP5_CMD_KEY_HI(&desc) = 0;
294 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
295 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
296
297 return ccp5_do_cmd(&desc, op->cmd_q);
298 }
299
300 static int ccp5_perform_xts_aes(struct ccp_op *op)
301 {
302 struct ccp5_desc desc;
303 union ccp_function function;
304 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
305
306 /* Zero out all the fields of the command desc */
307 memset(&desc, 0, Q_DESC_SIZE);
308
309 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
310
311 CCP5_CMD_SOC(&desc) = op->soc;
312 CCP5_CMD_IOC(&desc) = 1;
313 CCP5_CMD_INIT(&desc) = op->init;
314 CCP5_CMD_EOM(&desc) = op->eom;
315 CCP5_CMD_PROT(&desc) = 0;
316
317 function.raw = 0;
318 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
319 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
320 CCP5_CMD_FUNCTION(&desc) = function.raw;
321
322 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
323
324 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
325 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
326 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
327
328 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
329 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
330 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
331
332 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
333 CCP5_CMD_KEY_HI(&desc) = 0;
334 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
335 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
336
337 return ccp5_do_cmd(&desc, op->cmd_q);
338 }
339
340 static int ccp5_perform_sha(struct ccp_op *op)
341 {
342 struct ccp5_desc desc;
343 union ccp_function function;
344
345 /* Zero out all the fields of the command desc */
346 memset(&desc, 0, Q_DESC_SIZE);
347
348 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
349
350 CCP5_CMD_SOC(&desc) = op->soc;
351 CCP5_CMD_IOC(&desc) = 1;
352 CCP5_CMD_INIT(&desc) = 1;
353 CCP5_CMD_EOM(&desc) = op->eom;
354 CCP5_CMD_PROT(&desc) = 0;
355
356 function.raw = 0;
357 CCP_SHA_TYPE(&function) = op->u.sha.type;
358 CCP5_CMD_FUNCTION(&desc) = function.raw;
359
360 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
361
362 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
363 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
364 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
365
366 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
367
368 if (op->eom) {
369 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
370 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
371 } else {
372 CCP5_CMD_SHA_LO(&desc) = 0;
373 CCP5_CMD_SHA_HI(&desc) = 0;
374 }
375
376 return ccp5_do_cmd(&desc, op->cmd_q);
377 }
378
379 static int ccp5_perform_rsa(struct ccp_op *op)
380 {
381 struct ccp5_desc desc;
382 union ccp_function function;
383
384 /* Zero out all the fields of the command desc */
385 memset(&desc, 0, Q_DESC_SIZE);
386
387 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
388
389 CCP5_CMD_SOC(&desc) = op->soc;
390 CCP5_CMD_IOC(&desc) = 1;
391 CCP5_CMD_INIT(&desc) = 0;
392 CCP5_CMD_EOM(&desc) = 1;
393 CCP5_CMD_PROT(&desc) = 0;
394
395 function.raw = 0;
396 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
397 CCP5_CMD_FUNCTION(&desc) = function.raw;
398
399 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
400
401 /* Source is from external memory */
402 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
403 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
404 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
405
406 /* Destination is in external memory */
407 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
408 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
409 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
410
411 /* Key (Exponent) is in external memory */
412 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
413 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
414 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
415
416 return ccp5_do_cmd(&desc, op->cmd_q);
417 }
418
419 static int ccp5_perform_passthru(struct ccp_op *op)
420 {
421 struct ccp5_desc desc;
422 union ccp_function function;
423 struct ccp_dma_info *saddr = &op->src.u.dma;
424 struct ccp_dma_info *daddr = &op->dst.u.dma;
425
426 memset(&desc, 0, Q_DESC_SIZE);
427
428 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
429
430 CCP5_CMD_SOC(&desc) = 0;
431 CCP5_CMD_IOC(&desc) = 1;
432 CCP5_CMD_INIT(&desc) = 0;
433 CCP5_CMD_EOM(&desc) = op->eom;
434 CCP5_CMD_PROT(&desc) = 0;
435
436 function.raw = 0;
437 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
438 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
439 CCP5_CMD_FUNCTION(&desc) = function.raw;
440
441 /* Length of source data is always 256 bytes */
442 if (op->src.type == CCP_MEMTYPE_SYSTEM)
443 CCP5_CMD_LEN(&desc) = saddr->length;
444 else
445 CCP5_CMD_LEN(&desc) = daddr->length;
446
447 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
448 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
449 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
450 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
451
452 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
453 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
454 } else {
455 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
456
457 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
458 CCP5_CMD_SRC_HI(&desc) = 0;
459 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
460 }
461
462 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
463 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
464 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
465 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
466 } else {
467 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
468
469 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
470 CCP5_CMD_DST_HI(&desc) = 0;
471 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
472 }
473
474 return ccp5_do_cmd(&desc, op->cmd_q);
475 }
476
477 static int ccp5_perform_ecc(struct ccp_op *op)
478 {
479 struct ccp5_desc desc;
480 union ccp_function function;
481
482 /* Zero out all the fields of the command desc */
483 memset(&desc, 0, Q_DESC_SIZE);
484
485 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
486
487 CCP5_CMD_SOC(&desc) = 0;
488 CCP5_CMD_IOC(&desc) = 1;
489 CCP5_CMD_INIT(&desc) = 0;
490 CCP5_CMD_EOM(&desc) = 1;
491 CCP5_CMD_PROT(&desc) = 0;
492
493 function.raw = 0;
494 function.ecc.mode = op->u.ecc.function;
495 CCP5_CMD_FUNCTION(&desc) = function.raw;
496
497 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
498
499 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
500 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
501 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
502
503 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
504 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
505 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
506
507 return ccp5_do_cmd(&desc, op->cmd_q);
508 }
509
510 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
511 {
512 int q_mask = 1 << cmd_q->id;
513 int queues = 0;
514 int j;
515
516 /* Build a bit mask to know which LSBs this queue has access to.
517 * Don't bother with segment 0 as it has special privileges.
518 */
519 for (j = 1; j < MAX_LSB_CNT; j++) {
520 if (status & q_mask)
521 bitmap_set(cmd_q->lsbmask, j, 1);
522 status >>= LSB_REGION_WIDTH;
523 }
524 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
525 dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
526 cmd_q->id, queues);
527
528 return queues ? 0 : -EINVAL;
529 }
530
531
532 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
533 int lsb_cnt, int n_lsbs,
534 unsigned long *lsb_pub)
535 {
536 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
537 int bitno;
538 int qlsb_wgt;
539 int i;
540
541 /* For each queue:
542 * If the count of potential LSBs available to a queue matches the
543 * ordinal given to us in lsb_cnt:
544 * Copy the mask of possible LSBs for this queue into "qlsb";
545 * For each bit in qlsb, see if the corresponding bit in the
546 * aggregation mask is set; if so, we have a match.
547 * If we have a match, clear the bit in the aggregation to
548 * mark it as no longer available.
549 * If there is no match, clear the bit in qlsb and keep looking.
550 */
551 for (i = 0; i < ccp->cmd_q_count; i++) {
552 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
553
554 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
555
556 if (qlsb_wgt == lsb_cnt) {
557 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
558
559 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
560 while (bitno < MAX_LSB_CNT) {
561 if (test_bit(bitno, lsb_pub)) {
562 /* We found an available LSB
563 * that this queue can access
564 */
565 cmd_q->lsb = bitno;
566 bitmap_clear(lsb_pub, bitno, 1);
567 dev_info(ccp->dev,
568 "Queue %d gets LSB %d\n",
569 i, bitno);
570 break;
571 }
572 bitmap_clear(qlsb, bitno, 1);
573 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
574 }
575 if (bitno >= MAX_LSB_CNT)
576 return -EINVAL;
577 n_lsbs--;
578 }
579 }
580 return n_lsbs;
581 }
582
583 /* For each queue, from the most- to least-constrained:
584 * find an LSB that can be assigned to the queue. If there are N queues that
585 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
586 * dedicated LSB. Remaining LSB regions become a shared resource.
587 * If we have fewer LSBs than queues, all LSB regions become shared resources.
588 */
589 static int ccp_assign_lsbs(struct ccp_device *ccp)
590 {
591 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
592 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
593 int n_lsbs = 0;
594 int bitno;
595 int i, lsb_cnt;
596 int rc = 0;
597
598 bitmap_zero(lsb_pub, MAX_LSB_CNT);
599
600 /* Create an aggregate bitmap to get a total count of available LSBs */
601 for (i = 0; i < ccp->cmd_q_count; i++)
602 bitmap_or(lsb_pub,
603 lsb_pub, ccp->cmd_q[i].lsbmask,
604 MAX_LSB_CNT);
605
606 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
607
608 if (n_lsbs >= ccp->cmd_q_count) {
609 /* We have enough LSBS to give every queue a private LSB.
610 * Brute force search to start with the queues that are more
611 * constrained in LSB choice. When an LSB is privately
612 * assigned, it is removed from the public mask.
613 * This is an ugly N squared algorithm with some optimization.
614 */
615 for (lsb_cnt = 1;
616 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
617 lsb_cnt++) {
618 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
619 lsb_pub);
620 if (rc < 0)
621 return -EINVAL;
622 n_lsbs = rc;
623 }
624 }
625
626 rc = 0;
627 /* What's left of the LSBs, according to the public mask, now become
628 * shared. Any zero bits in the lsb_pub mask represent an LSB region
629 * that can't be used as a shared resource, so mark the LSB slots for
630 * them as "in use".
631 */
632 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
633
634 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
635 while (bitno < MAX_LSB_CNT) {
636 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
637 bitmap_set(qlsb, bitno, 1);
638 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
639 }
640
641 return rc;
642 }
643
644 static int ccp5_init(struct ccp_device *ccp)
645 {
646 struct device *dev = ccp->dev;
647 struct ccp_cmd_queue *cmd_q;
648 struct dma_pool *dma_pool;
649 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
650 unsigned int qmr, qim, i;
651 u64 status;
652 u32 status_lo, status_hi;
653 int ret;
654
655 /* Find available queues */
656 qim = 0;
657 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
658 for (i = 0; i < MAX_HW_QUEUES; i++) {
659
660 if (!(qmr & (1 << i)))
661 continue;
662
663 /* Allocate a dma pool for this queue */
664 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
665 ccp->name, i);
666 dma_pool = dma_pool_create(dma_pool_name, dev,
667 CCP_DMAPOOL_MAX_SIZE,
668 CCP_DMAPOOL_ALIGN, 0);
669 if (!dma_pool) {
670 dev_err(dev, "unable to allocate dma pool\n");
671 ret = -ENOMEM;
672 }
673
674 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
675 ccp->cmd_q_count++;
676
677 cmd_q->ccp = ccp;
678 cmd_q->id = i;
679 cmd_q->dma_pool = dma_pool;
680 mutex_init(&cmd_q->q_mutex);
681
682 /* Page alignment satisfies our needs for N <= 128 */
683 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
684 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
685 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
686 &cmd_q->qbase_dma,
687 GFP_KERNEL);
688 if (!cmd_q->qbase) {
689 dev_err(dev, "unable to allocate command queue\n");
690 ret = -ENOMEM;
691 goto e_pool;
692 }
693
694 cmd_q->qidx = 0;
695 /* Preset some register values and masks that are queue
696 * number dependent
697 */
698 cmd_q->reg_control = ccp->io_regs +
699 CMD5_Q_STATUS_INCR * (i + 1);
700 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
701 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
702 cmd_q->reg_int_enable = cmd_q->reg_control +
703 CMD5_Q_INT_ENABLE_BASE;
704 cmd_q->reg_interrupt_status = cmd_q->reg_control +
705 CMD5_Q_INTERRUPT_STATUS_BASE;
706 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
707 cmd_q->reg_int_status = cmd_q->reg_control +
708 CMD5_Q_INT_STATUS_BASE;
709 cmd_q->reg_dma_status = cmd_q->reg_control +
710 CMD5_Q_DMA_STATUS_BASE;
711 cmd_q->reg_dma_read_status = cmd_q->reg_control +
712 CMD5_Q_DMA_READ_STATUS_BASE;
713 cmd_q->reg_dma_write_status = cmd_q->reg_control +
714 CMD5_Q_DMA_WRITE_STATUS_BASE;
715
716 init_waitqueue_head(&cmd_q->int_queue);
717
718 dev_dbg(dev, "queue #%u available\n", i);
719 }
720 if (ccp->cmd_q_count == 0) {
721 dev_notice(dev, "no command queues available\n");
722 ret = -EIO;
723 goto e_pool;
724 }
725 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
726
727 /* Turn off the queues and disable interrupts until ready */
728 for (i = 0; i < ccp->cmd_q_count; i++) {
729 cmd_q = &ccp->cmd_q[i];
730
731 cmd_q->qcontrol = 0; /* Start with nothing */
732 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
733
734 /* Disable the interrupts */
735 iowrite32(0x00, cmd_q->reg_int_enable);
736 ioread32(cmd_q->reg_int_status);
737 ioread32(cmd_q->reg_status);
738
739 /* Clear the interrupts */
740 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
741 }
742
743 dev_dbg(dev, "Requesting an IRQ...\n");
744 /* Request an irq */
745 ret = ccp->get_irq(ccp);
746 if (ret) {
747 dev_err(dev, "unable to allocate an IRQ\n");
748 goto e_pool;
749 }
750
751 /* Initialize the queue used to suspend */
752 init_waitqueue_head(&ccp->suspend_queue);
753
754 dev_dbg(dev, "Loading LSB map...\n");
755 /* Copy the private LSB mask to the public registers */
756 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
757 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
758 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
759 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
760 status = ((u64)status_hi<<30) | (u64)status_lo;
761
762 dev_dbg(dev, "Configuring virtual queues...\n");
763 /* Configure size of each virtual queue accessible to host */
764 for (i = 0; i < ccp->cmd_q_count; i++) {
765 u32 dma_addr_lo;
766 u32 dma_addr_hi;
767
768 cmd_q = &ccp->cmd_q[i];
769
770 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
771 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
772
773 cmd_q->qdma_tail = cmd_q->qbase_dma;
774 dma_addr_lo = low_address(cmd_q->qdma_tail);
775 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
776 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
777
778 dma_addr_hi = high_address(cmd_q->qdma_tail);
779 cmd_q->qcontrol |= (dma_addr_hi << 16);
780 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
781
782 /* Find the LSB regions accessible to the queue */
783 ccp_find_lsb_regions(cmd_q, status);
784 cmd_q->lsb = -1; /* Unassigned value */
785 }
786
787 dev_dbg(dev, "Assigning LSBs...\n");
788 ret = ccp_assign_lsbs(ccp);
789 if (ret) {
790 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
791 goto e_irq;
792 }
793
794 /* Optimization: pre-allocate LSB slots for each queue */
795 for (i = 0; i < ccp->cmd_q_count; i++) {
796 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
797 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
798 }
799
800 dev_dbg(dev, "Starting threads...\n");
801 /* Create a kthread for each queue */
802 for (i = 0; i < ccp->cmd_q_count; i++) {
803 struct task_struct *kthread;
804
805 cmd_q = &ccp->cmd_q[i];
806
807 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
808 "%s-q%u", ccp->name, cmd_q->id);
809 if (IS_ERR(kthread)) {
810 dev_err(dev, "error creating queue thread (%ld)\n",
811 PTR_ERR(kthread));
812 ret = PTR_ERR(kthread);
813 goto e_kthread;
814 }
815
816 cmd_q->kthread = kthread;
817 wake_up_process(kthread);
818 }
819
820 dev_dbg(dev, "Enabling interrupts...\n");
821 /* Enable interrupts */
822 for (i = 0; i < ccp->cmd_q_count; i++) {
823 cmd_q = &ccp->cmd_q[i];
824 iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable);
825 }
826
827 dev_dbg(dev, "Registering device...\n");
828 /* Put this on the unit list to make it available */
829 ccp_add_device(ccp);
830
831 ret = ccp_register_rng(ccp);
832 if (ret)
833 goto e_kthread;
834
835 /* Register the DMA engine support */
836 ret = ccp_dmaengine_register(ccp);
837 if (ret)
838 goto e_kthread;
839
840 return 0;
841
842 e_kthread:
843 for (i = 0; i < ccp->cmd_q_count; i++)
844 if (ccp->cmd_q[i].kthread)
845 kthread_stop(ccp->cmd_q[i].kthread);
846
847 e_irq:
848 ccp->free_irq(ccp);
849
850 e_pool:
851 for (i = 0; i < ccp->cmd_q_count; i++)
852 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
853
854 return ret;
855 }
856
857 static void ccp5_destroy(struct ccp_device *ccp)
858 {
859 struct device *dev = ccp->dev;
860 struct ccp_cmd_queue *cmd_q;
861 struct ccp_cmd *cmd;
862 unsigned int i;
863
864 /* Unregister the DMA engine */
865 ccp_dmaengine_unregister(ccp);
866
867 /* Unregister the RNG */
868 ccp_unregister_rng(ccp);
869
870 /* Remove this device from the list of available units first */
871 ccp_del_device(ccp);
872
873 /* Disable and clear interrupts */
874 for (i = 0; i < ccp->cmd_q_count; i++) {
875 cmd_q = &ccp->cmd_q[i];
876
877 /* Turn off the run bit */
878 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
879
880 /* Disable the interrupts */
881 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
882
883 /* Clear the interrupt status */
884 iowrite32(0x00, cmd_q->reg_int_enable);
885 ioread32(cmd_q->reg_int_status);
886 ioread32(cmd_q->reg_status);
887 }
888
889 /* Stop the queue kthreads */
890 for (i = 0; i < ccp->cmd_q_count; i++)
891 if (ccp->cmd_q[i].kthread)
892 kthread_stop(ccp->cmd_q[i].kthread);
893
894 ccp->free_irq(ccp);
895
896 for (i = 0; i < ccp->cmd_q_count; i++) {
897 cmd_q = &ccp->cmd_q[i];
898 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
899 cmd_q->qbase_dma);
900 }
901
902 /* Flush the cmd and backlog queue */
903 while (!list_empty(&ccp->cmd)) {
904 /* Invoke the callback directly with an error code */
905 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
906 list_del(&cmd->entry);
907 cmd->callback(cmd->data, -ENODEV);
908 }
909 while (!list_empty(&ccp->backlog)) {
910 /* Invoke the callback directly with an error code */
911 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
912 list_del(&cmd->entry);
913 cmd->callback(cmd->data, -ENODEV);
914 }
915 }
916
917 static irqreturn_t ccp5_irq_handler(int irq, void *data)
918 {
919 struct device *dev = data;
920 struct ccp_device *ccp = dev_get_drvdata(dev);
921 u32 status;
922 unsigned int i;
923
924 for (i = 0; i < ccp->cmd_q_count; i++) {
925 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
926
927 status = ioread32(cmd_q->reg_interrupt_status);
928
929 if (status) {
930 cmd_q->int_status = status;
931 cmd_q->q_status = ioread32(cmd_q->reg_status);
932 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
933
934 /* On error, only save the first error value */
935 if ((status & INT_ERROR) && !cmd_q->cmd_error)
936 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
937
938 cmd_q->int_rcvd = 1;
939
940 /* Acknowledge the interrupt and wake the kthread */
941 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
942 wake_up_interruptible(&cmd_q->int_queue);
943 }
944 }
945
946 return IRQ_HANDLED;
947 }
948
949 static void ccp5_config(struct ccp_device *ccp)
950 {
951 /* Public side */
952 iowrite32(0x00001249, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
953 }
954
955 static void ccp5other_config(struct ccp_device *ccp)
956 {
957 int i;
958 u32 rnd;
959
960 /* We own all of the queues on the NTB CCP */
961
962 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
963 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
964 for (i = 0; i < 12; i++) {
965 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
966 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
967 }
968
969 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
970 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
971 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
972
973 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
974 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
975
976 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
977
978 ccp5_config(ccp);
979 }
980
981 /* Version 5 adds some function, but is essentially the same as v5 */
982 static const struct ccp_actions ccp5_actions = {
983 .aes = ccp5_perform_aes,
984 .xts_aes = ccp5_perform_xts_aes,
985 .sha = ccp5_perform_sha,
986 .rsa = ccp5_perform_rsa,
987 .passthru = ccp5_perform_passthru,
988 .ecc = ccp5_perform_ecc,
989 .sballoc = ccp_lsb_alloc,
990 .sbfree = ccp_lsb_free,
991 .init = ccp5_init,
992 .destroy = ccp5_destroy,
993 .get_free_slots = ccp5_get_free_slots,
994 .irqhandler = ccp5_irq_handler,
995 };
996
997 struct ccp_vdata ccpv5 = {
998 .version = CCP_VERSION(5, 0),
999 .setup = ccp5_config,
1000 .perform = &ccp5_actions,
1001 .bar = 2,
1002 .offset = 0x0,
1003 };
1004
1005 struct ccp_vdata ccpv5other = {
1006 .version = CCP_VERSION(5, 0),
1007 .setup = ccp5other_config,
1008 .perform = &ccp5_actions,
1009 .bar = 2,
1010 .offset = 0x0,
1011 };
Linux kernel - Deliverables
This page took 0.050462 seconds and 5 git commands to generate.