2 * AMD Cryptographic Coprocessor (CCP) driver
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
17 #include <linux/device.h>
18 #include <linux/pci.h>
19 #include <linux/spinlock.h>
20 #include <linux/mutex.h>
21 #include <linux/list.h>
22 #include <linux/wait.h>
23 #include <linux/dmapool.h>
24 #include <linux/hw_random.h>
25 #include <linux/bitops.h>
26 #include <linux/interrupt.h>
27 #include <linux/irqreturn.h>
28 #include <linux/dmaengine.h>
30 #define MAX_CCP_NAME_LEN 16
31 #define MAX_DMAPOOL_NAME_LEN 32
33 #define MAX_HW_QUEUES 5
34 #define MAX_CMD_QLEN 100
36 #define TRNG_RETRIES 10
38 #define CACHE_NONE 0x00
39 #define CACHE_WB_NO_ALLOC 0xb7
41 /****** Register Mappings ******/
42 #define Q_MASK_REG 0x000
43 #define TRNG_OUT_REG 0x00c
44 #define IRQ_MASK_REG 0x040
45 #define IRQ_STATUS_REG 0x200
47 #define DEL_CMD_Q_JOB 0x124
48 #define DEL_Q_ACTIVE 0x00000200
49 #define DEL_Q_ID_SHIFT 6
51 #define CMD_REQ0 0x180
52 #define CMD_REQ_INCR 0x04
54 #define CMD_Q_STATUS_BASE 0x210
55 #define CMD_Q_INT_STATUS_BASE 0x214
56 #define CMD_Q_STATUS_INCR 0x20
58 #define CMD_Q_CACHE_BASE 0x228
59 #define CMD_Q_CACHE_INC 0x20
61 #define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
62 #define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f)
64 /****** REQ0 Related Values ******/
65 #define REQ0_WAIT_FOR_WRITE 0x00000004
66 #define REQ0_INT_ON_COMPLETE 0x00000002
67 #define REQ0_STOP_ON_COMPLETE 0x00000001
69 #define REQ0_CMD_Q_SHIFT 9
70 #define REQ0_JOBID_SHIFT 3
72 /****** REQ1 Related Values ******/
73 #define REQ1_PROTECT_SHIFT 27
74 #define REQ1_ENGINE_SHIFT 23
75 #define REQ1_KEY_KSB_SHIFT 2
77 #define REQ1_EOM 0x00000002
78 #define REQ1_INIT 0x00000001
80 /* AES Related Values */
81 #define REQ1_AES_TYPE_SHIFT 21
82 #define REQ1_AES_MODE_SHIFT 18
83 #define REQ1_AES_ACTION_SHIFT 17
84 #define REQ1_AES_CFB_SIZE_SHIFT 10
86 /* XTS-AES Related Values */
87 #define REQ1_XTS_AES_SIZE_SHIFT 10
89 /* SHA Related Values */
90 #define REQ1_SHA_TYPE_SHIFT 21
92 /* RSA Related Values */
93 #define REQ1_RSA_MOD_SIZE_SHIFT 10
95 /* Pass-Through Related Values */
96 #define REQ1_PT_BW_SHIFT 12
97 #define REQ1_PT_BS_SHIFT 10
99 /* ECC Related Values */
100 #define REQ1_ECC_AFFINE_CONVERT 0x00200000
101 #define REQ1_ECC_FUNCTION_SHIFT 18
103 /****** REQ4 Related Values ******/
104 #define REQ4_KSB_SHIFT 18
105 #define REQ4_MEMTYPE_SHIFT 16
107 /****** REQ6 Related Values ******/
108 #define REQ6_MEMTYPE_SHIFT 16
110 /****** Key Storage Block ******/
113 #define KSB_COUNT (KSB_END - KSB_START + 1)
114 #define CCP_SB_BITS 256
116 #define CCP_JOBID_MASK 0x0000003f
118 #define CCP_DMAPOOL_MAX_SIZE 64
119 #define CCP_DMAPOOL_ALIGN BIT(5)
121 #define CCP_REVERSE_BUF_SIZE 64
123 #define CCP_AES_KEY_SB_COUNT 1
124 #define CCP_AES_CTX_SB_COUNT 1
126 #define CCP_XTS_AES_KEY_SB_COUNT 1
127 #define CCP_XTS_AES_CTX_SB_COUNT 1
129 #define CCP_SHA_SB_COUNT 1
131 #define CCP_RSA_MAX_WIDTH 4096
133 #define CCP_PASSTHRU_BLOCKSIZE 256
134 #define CCP_PASSTHRU_MASKSIZE 32
135 #define CCP_PASSTHRU_SB_COUNT 1
137 #define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */
138 #define CCP_ECC_MAX_OPERANDS 6
139 #define CCP_ECC_MAX_OUTPUTS 3
140 #define CCP_ECC_SRC_BUF_SIZE 448
141 #define CCP_ECC_DST_BUF_SIZE 192
142 #define CCP_ECC_OPERAND_SIZE 64
143 #define CCP_ECC_OUTPUT_SIZE 64
144 #define CCP_ECC_RESULT_OFFSET 60
145 #define CCP_ECC_RESULT_SUCCESS 0x0001
147 #define CCP_SB_BYTES 32
154 struct list_head entry
;
156 struct ccp_cmd ccp_cmd
;
159 struct ccp_dma_desc
{
160 struct list_head entry
;
162 struct ccp_device
*ccp
;
164 struct list_head pending
;
165 struct list_head active
;
167 enum dma_status status
;
168 struct dma_async_tx_descriptor tx_desc
;
172 struct ccp_dma_chan
{
173 struct ccp_device
*ccp
;
176 struct list_head pending
;
177 struct list_head active
;
178 struct list_head complete
;
180 struct tasklet_struct cleanup_tasklet
;
182 enum dma_status status
;
183 struct dma_chan dma_chan
;
186 struct ccp_cmd_queue
{
187 struct ccp_device
*ccp
;
189 /* Queue identifier */
193 struct dma_pool
*dma_pool
;
195 /* Per-queue reserved storage block(s) */
199 /* Queue processing thread */
200 struct task_struct
*kthread
;
202 unsigned int suspended
;
204 /* Number of free command slots available */
205 unsigned int free_slots
;
207 /* Interrupt masks */
211 /* Register addresses for queue */
212 void __iomem
*reg_status
;
213 void __iomem
*reg_int_status
;
215 /* Status values from job */
221 /* Interrupt wait queue */
222 wait_queue_head_t int_queue
;
223 unsigned int int_rcvd
;
224 } ____cacheline_aligned
;
227 struct list_head entry
;
229 struct ccp_vdata
*vdata
;
231 char name
[MAX_CCP_NAME_LEN
];
232 char rngname
[MAX_CCP_NAME_LEN
];
236 /* Bus specific device information
239 int (*get_irq
)(struct ccp_device
*ccp
);
240 void (*free_irq
)(struct ccp_device
*ccp
);
243 /* I/O area used for device communication. The register mapping
244 * starts at an offset into the mapped bar.
245 * The CMD_REQx registers and the Delete_Cmd_Queue_Job register
246 * need to be protected while a command queue thread is accessing
249 struct mutex req_mutex ____cacheline_aligned
;
250 void __iomem
*io_map
;
251 void __iomem
*io_regs
;
253 /* Master lists that all cmds are queued on. Because there can be
254 * more than one CCP command queue that can process a cmd a separate
255 * backlog list is neeeded so that the backlog completion call
256 * completes before the cmd is available for execution.
258 spinlock_t cmd_lock ____cacheline_aligned
;
259 unsigned int cmd_count
;
260 struct list_head cmd
;
261 struct list_head backlog
;
263 /* The command queues. These represent the queues available on the
264 * CCP that are available for processing cmds
266 struct ccp_cmd_queue cmd_q
[MAX_HW_QUEUES
];
267 unsigned int cmd_q_count
;
269 /* Support for the CCP True RNG
272 unsigned int hwrng_retries
;
274 /* Support for the CCP DMA capabilities
276 struct dma_device dma_dev
;
277 struct ccp_dma_chan
*ccp_dma_chan
;
278 struct kmem_cache
*dma_cmd_cache
;
279 struct kmem_cache
*dma_desc_cache
;
281 /* A counter used to generate job-ids for cmds submitted to the CCP
283 atomic_t current_id ____cacheline_aligned
;
285 /* The v3 CCP uses key storage blocks (SB) to maintain context for
286 * certain operations. To prevent multiple cmds from using the same
287 * SB range a command queue reserves an SB range for the duration of
288 * the cmd. Each queue, will however, reserve 2 SB blocks for
289 * operations that only require single SB entries (eg. AES context/iv
290 * and key) in order to avoid allocation contention. This will reserve
291 * at most 10 SB entries, leaving 40 SB entries available for dynamic
294 * The v5 CCP Local Storage Block (LSB) is broken up into 8
295 * memrory ranges, each of which can be enabled for access by one
296 * or more queues. Device initialization takes this into account,
297 * and attempts to assign one region for exclusive use by each
298 * available queue; the rest are then aggregated as "public" use.
299 * If there are fewer regions than queues, all regions are shared
300 * amongst all queues.
302 struct mutex sb_mutex ____cacheline_aligned
;
303 DECLARE_BITMAP(sb
, KSB_COUNT
);
304 wait_queue_head_t sb_queue
;
305 unsigned int sb_avail
;
306 unsigned int sb_count
;
309 /* Suspend support */
310 unsigned int suspending
;
311 wait_queue_head_t suspend_queue
;
313 /* DMA caching attribute support */
314 unsigned int axcache
;
318 CCP_MEMTYPE_SYSTEM
= 0,
324 struct ccp_dma_info
{
328 enum dma_data_direction dir
;
331 struct ccp_dm_workarea
{
333 struct dma_pool
*dma_pool
;
337 struct ccp_dma_info dma
;
340 struct ccp_sg_workarea
{
341 struct scatterlist
*sg
;
344 struct scatterlist
*dma_sg
;
345 struct device
*dma_dev
;
346 unsigned int dma_count
;
347 enum dma_data_direction dma_dir
;
349 unsigned int sg_used
;
355 struct ccp_sg_workarea sg_wa
;
356 struct ccp_dm_workarea dm_wa
;
360 enum ccp_memtype type
;
362 struct ccp_dma_info dma
;
368 enum ccp_aes_type type
;
369 enum ccp_aes_mode mode
;
370 enum ccp_aes_action action
;
373 struct ccp_xts_aes_op
{
374 enum ccp_aes_action action
;
375 enum ccp_xts_aes_unit_size unit_size
;
379 enum ccp_sha_type type
;
388 struct ccp_passthru_op
{
389 enum ccp_passthru_bitwise bit_mod
;
390 enum ccp_passthru_byteswap byte_swap
;
394 enum ccp_ecc_function function
;
398 struct ccp_cmd_queue
*cmd_q
;
412 struct ccp_aes_op aes
;
413 struct ccp_xts_aes_op xts
;
414 struct ccp_sha_op sha
;
415 struct ccp_rsa_op rsa
;
416 struct ccp_passthru_op passthru
;
417 struct ccp_ecc_op ecc
;
421 static inline u32
ccp_addr_lo(struct ccp_dma_info
*info
)
423 return lower_32_bits(info
->address
+ info
->offset
);
426 static inline u32
ccp_addr_hi(struct ccp_dma_info
*info
)
428 return upper_32_bits(info
->address
+ info
->offset
) & 0x0000ffff;
431 int ccp_pci_init(void);
432 void ccp_pci_exit(void);
434 int ccp_platform_init(void);
435 void ccp_platform_exit(void);
437 void ccp_add_device(struct ccp_device
*ccp
);
438 void ccp_del_device(struct ccp_device
*ccp
);
440 struct ccp_device
*ccp_alloc_struct(struct device
*dev
);
441 bool ccp_queues_suspended(struct ccp_device
*ccp
);
442 int ccp_cmd_queue_thread(void *data
);
444 int ccp_run_cmd(struct ccp_cmd_queue
*cmd_q
, struct ccp_cmd
*cmd
);
446 int ccp_dmaengine_register(struct ccp_device
*ccp
);
447 void ccp_dmaengine_unregister(struct ccp_device
*ccp
);
449 /* Structure for computation functions that are device-specific */
451 int (*aes
)(struct ccp_op
*);
452 int (*xts_aes
)(struct ccp_op
*);
453 int (*sha
)(struct ccp_op
*);
454 int (*rsa
)(struct ccp_op
*);
455 int (*passthru
)(struct ccp_op
*);
456 int (*ecc
)(struct ccp_op
*);
457 u32 (*sballoc
)(struct ccp_cmd_queue
*, unsigned int);
458 void (*sbfree
)(struct ccp_cmd_queue
*, unsigned int,
460 int (*init
)(struct ccp_device
*);
461 void (*destroy
)(struct ccp_device
*);
462 irqreturn_t (*irqhandler
)(int, void *);
465 /* Structure to hold CCP version-specific values */
467 unsigned int version
;
468 int (*init
)(struct ccp_device
*);
469 const struct ccp_actions
*perform
;
470 const unsigned int bar
;
471 const unsigned int offset
;
474 extern struct ccp_vdata ccpv3
;