2 * Copyright 2014 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
13 #include <linux/interrupt.h>
14 #include <linux/semaphore.h>
15 #include <linux/device.h>
16 #include <linux/types.h>
17 #include <linux/cdev.h>
18 #include <linux/pid.h>
20 #include <linux/pci.h>
22 #include <asm/cputable.h>
25 #include <misc/cxl-base.h>
27 #include <uapi/misc/cxl.h>
29 extern uint cxl_verbose
;
34 * Bump version each time a user API change is made, whether it is
35 * backwards compatible ot not.
37 #define CXL_API_VERSION 2
38 #define CXL_API_VERSION_COMPATIBLE 1
41 * Opaque types to avoid accidentally passing registers for the wrong MMIO
43 * At the end of the day, I'm not married to using typedef here, but it might
44 * (and has!) help avoid bugs like mixing up CXL_PSL_CtxTime and
45 * CXL_PSL_CtxTime_An, or calling cxl_p1n_write instead of cxl_p1_write.
47 * I'm quite happy if these are changed back to #defines before upstreaming, it
48 * should be little more than a regexp search+replace operation in this file.
59 #define cxl_reg_off(reg) \
62 /* Memory maps. Ref CXL Appendix A */
64 /* PSL Privilege 1 Memory Map */
65 /* Configuration and Control area */
66 static const cxl_p1_reg_t CXL_PSL_CtxTime
= {0x0000};
67 static const cxl_p1_reg_t CXL_PSL_ErrIVTE
= {0x0008};
68 static const cxl_p1_reg_t CXL_PSL_KEY1
= {0x0010};
69 static const cxl_p1_reg_t CXL_PSL_KEY2
= {0x0018};
70 static const cxl_p1_reg_t CXL_PSL_Control
= {0x0020};
72 static const cxl_p1_reg_t CXL_PSL_DLCNTL
= {0x0060};
73 static const cxl_p1_reg_t CXL_PSL_DLADDR
= {0x0068};
75 /* PSL Lookaside Buffer Management Area */
76 static const cxl_p1_reg_t CXL_PSL_LBISEL
= {0x0080};
77 static const cxl_p1_reg_t CXL_PSL_SLBIE
= {0x0088};
78 static const cxl_p1_reg_t CXL_PSL_SLBIA
= {0x0090};
79 static const cxl_p1_reg_t CXL_PSL_TLBIE
= {0x00A0};
80 static const cxl_p1_reg_t CXL_PSL_TLBIA
= {0x00A8};
81 static const cxl_p1_reg_t CXL_PSL_AFUSEL
= {0x00B0};
83 /* 0x00C0:7EFF Implementation dependent area */
84 static const cxl_p1_reg_t CXL_PSL_FIR1
= {0x0100};
85 static const cxl_p1_reg_t CXL_PSL_FIR2
= {0x0108};
86 static const cxl_p1_reg_t CXL_PSL_Timebase
= {0x0110};
87 static const cxl_p1_reg_t CXL_PSL_VERSION
= {0x0118};
88 static const cxl_p1_reg_t CXL_PSL_RESLCKTO
= {0x0128};
89 static const cxl_p1_reg_t CXL_PSL_TB_CTLSTAT
= {0x0140};
90 static const cxl_p1_reg_t CXL_PSL_FIR_CNTL
= {0x0148};
91 static const cxl_p1_reg_t CXL_PSL_DSNDCTL
= {0x0150};
92 static const cxl_p1_reg_t CXL_PSL_SNWRALLOC
= {0x0158};
93 static const cxl_p1_reg_t CXL_PSL_TRACE
= {0x0170};
94 /* 0x7F00:7FFF Reserved PCIe MSI-X Pending Bit Array area */
95 /* 0x8000:FFFF Reserved PCIe MSI-X Table Area */
97 /* PSL Slice Privilege 1 Memory Map */
98 /* Configuration Area */
99 static const cxl_p1n_reg_t CXL_PSL_SR_An
= {0x00};
100 static const cxl_p1n_reg_t CXL_PSL_LPID_An
= {0x08};
101 static const cxl_p1n_reg_t CXL_PSL_AMBAR_An
= {0x10};
102 static const cxl_p1n_reg_t CXL_PSL_SPOffset_An
= {0x18};
103 static const cxl_p1n_reg_t CXL_PSL_ID_An
= {0x20};
104 static const cxl_p1n_reg_t CXL_PSL_SERR_An
= {0x28};
105 /* Memory Management and Lookaside Buffer Management */
106 static const cxl_p1n_reg_t CXL_PSL_SDR_An
= {0x30};
107 static const cxl_p1n_reg_t CXL_PSL_AMOR_An
= {0x38};
109 static const cxl_p1n_reg_t CXL_HAURP_An
= {0x80};
110 static const cxl_p1n_reg_t CXL_PSL_SPAP_An
= {0x88};
111 static const cxl_p1n_reg_t CXL_PSL_LLCMD_An
= {0x90};
113 static const cxl_p1n_reg_t CXL_PSL_SCNTL_An
= {0xA0};
114 static const cxl_p1n_reg_t CXL_PSL_CtxTime_An
= {0xA8};
115 static const cxl_p1n_reg_t CXL_PSL_IVTE_Offset_An
= {0xB0};
116 static const cxl_p1n_reg_t CXL_PSL_IVTE_Limit_An
= {0xB8};
117 /* 0xC0:FF Implementation Dependent Area */
118 static const cxl_p1n_reg_t CXL_PSL_FIR_SLICE_An
= {0xC0};
119 static const cxl_p1n_reg_t CXL_AFU_DEBUG_An
= {0xC8};
120 static const cxl_p1n_reg_t CXL_PSL_APCALLOC_A
= {0xD0};
121 static const cxl_p1n_reg_t CXL_PSL_COALLOC_A
= {0xD8};
122 static const cxl_p1n_reg_t CXL_PSL_RXCTL_A
= {0xE0};
123 static const cxl_p1n_reg_t CXL_PSL_SLICE_TRACE
= {0xE8};
125 /* PSL Slice Privilege 2 Memory Map */
126 /* Configuration and Control Area */
127 static const cxl_p2n_reg_t CXL_PSL_PID_TID_An
= {0x000};
128 static const cxl_p2n_reg_t CXL_CSRP_An
= {0x008};
129 static const cxl_p2n_reg_t CXL_AURP0_An
= {0x010};
130 static const cxl_p2n_reg_t CXL_AURP1_An
= {0x018};
131 static const cxl_p2n_reg_t CXL_SSTP0_An
= {0x020};
132 static const cxl_p2n_reg_t CXL_SSTP1_An
= {0x028};
133 static const cxl_p2n_reg_t CXL_PSL_AMR_An
= {0x030};
134 /* Segment Lookaside Buffer Management */
135 static const cxl_p2n_reg_t CXL_SLBIE_An
= {0x040};
136 static const cxl_p2n_reg_t CXL_SLBIA_An
= {0x048};
137 static const cxl_p2n_reg_t CXL_SLBI_Select_An
= {0x050};
138 /* Interrupt Registers */
139 static const cxl_p2n_reg_t CXL_PSL_DSISR_An
= {0x060};
140 static const cxl_p2n_reg_t CXL_PSL_DAR_An
= {0x068};
141 static const cxl_p2n_reg_t CXL_PSL_DSR_An
= {0x070};
142 static const cxl_p2n_reg_t CXL_PSL_TFC_An
= {0x078};
143 static const cxl_p2n_reg_t CXL_PSL_PEHandle_An
= {0x080};
144 static const cxl_p2n_reg_t CXL_PSL_ErrStat_An
= {0x088};
146 static const cxl_p2n_reg_t CXL_AFU_Cntl_An
= {0x090};
147 static const cxl_p2n_reg_t CXL_AFU_ERR_An
= {0x098};
148 /* Work Element Descriptor */
149 static const cxl_p2n_reg_t CXL_PSL_WED_An
= {0x0A0};
150 /* 0x0C0:FFF Implementation Dependent Area */
152 #define CXL_PSL_SPAP_Addr 0x0ffffffffffff000ULL
153 #define CXL_PSL_SPAP_Size 0x0000000000000ff0ULL
154 #define CXL_PSL_SPAP_Size_Shift 4
155 #define CXL_PSL_SPAP_V 0x0000000000000001ULL
157 /****** CXL_PSL_Control ****************************************************/
158 #define CXL_PSL_Control_tb 0x0000000000000001ULL
160 /****** CXL_PSL_DLCNTL *****************************************************/
161 #define CXL_PSL_DLCNTL_D (0x1ull << (63-28))
162 #define CXL_PSL_DLCNTL_C (0x1ull << (63-29))
163 #define CXL_PSL_DLCNTL_E (0x1ull << (63-30))
164 #define CXL_PSL_DLCNTL_S (0x1ull << (63-31))
165 #define CXL_PSL_DLCNTL_CE (CXL_PSL_DLCNTL_C | CXL_PSL_DLCNTL_E)
166 #define CXL_PSL_DLCNTL_DCES (CXL_PSL_DLCNTL_D | CXL_PSL_DLCNTL_CE | CXL_PSL_DLCNTL_S)
168 /****** CXL_PSL_SR_An ******************************************************/
169 #define CXL_PSL_SR_An_SF MSR_SF /* 64bit */
170 #define CXL_PSL_SR_An_TA (1ull << (63-1)) /* Tags active, GA1: 0 */
171 #define CXL_PSL_SR_An_HV MSR_HV /* Hypervisor, GA1: 0 */
172 #define CXL_PSL_SR_An_PR MSR_PR /* Problem state, GA1: 1 */
173 #define CXL_PSL_SR_An_ISL (1ull << (63-53)) /* Ignore Segment Large Page */
174 #define CXL_PSL_SR_An_TC (1ull << (63-54)) /* Page Table secondary hash */
175 #define CXL_PSL_SR_An_US (1ull << (63-56)) /* User state, GA1: X */
176 #define CXL_PSL_SR_An_SC (1ull << (63-58)) /* Segment Table secondary hash */
177 #define CXL_PSL_SR_An_R MSR_DR /* Relocate, GA1: 1 */
178 #define CXL_PSL_SR_An_MP (1ull << (63-62)) /* Master Process */
179 #define CXL_PSL_SR_An_LE (1ull << (63-63)) /* Little Endian */
181 /****** CXL_PSL_ID_An ****************************************************/
182 #define CXL_PSL_ID_An_F (1ull << (63-31))
183 #define CXL_PSL_ID_An_L (1ull << (63-30))
185 /****** CXL_PSL_SCNTL_An ****************************************************/
186 #define CXL_PSL_SCNTL_An_CR (0x1ull << (63-15))
187 /* Programming Modes: */
188 #define CXL_PSL_SCNTL_An_PM_MASK (0xffffull << (63-31))
189 #define CXL_PSL_SCNTL_An_PM_Shared (0x0000ull << (63-31))
190 #define CXL_PSL_SCNTL_An_PM_OS (0x0001ull << (63-31))
191 #define CXL_PSL_SCNTL_An_PM_Process (0x0002ull << (63-31))
192 #define CXL_PSL_SCNTL_An_PM_AFU (0x0004ull << (63-31))
193 #define CXL_PSL_SCNTL_An_PM_AFU_PBT (0x0104ull << (63-31))
194 /* Purge Status (ro) */
195 #define CXL_PSL_SCNTL_An_Ps_MASK (0x3ull << (63-39))
196 #define CXL_PSL_SCNTL_An_Ps_Pending (0x1ull << (63-39))
197 #define CXL_PSL_SCNTL_An_Ps_Complete (0x3ull << (63-39))
199 #define CXL_PSL_SCNTL_An_Pc (0x1ull << (63-48))
200 /* Suspend Status (ro) */
201 #define CXL_PSL_SCNTL_An_Ss_MASK (0x3ull << (63-55))
202 #define CXL_PSL_SCNTL_An_Ss_Pending (0x1ull << (63-55))
203 #define CXL_PSL_SCNTL_An_Ss_Complete (0x3ull << (63-55))
204 /* Suspend Control */
205 #define CXL_PSL_SCNTL_An_Sc (0x1ull << (63-63))
207 /* AFU Slice Enable Status (ro) */
208 #define CXL_AFU_Cntl_An_ES_MASK (0x7ull << (63-2))
209 #define CXL_AFU_Cntl_An_ES_Disabled (0x0ull << (63-2))
210 #define CXL_AFU_Cntl_An_ES_Enabled (0x4ull << (63-2))
211 /* AFU Slice Enable */
212 #define CXL_AFU_Cntl_An_E (0x1ull << (63-3))
213 /* AFU Slice Reset status (ro) */
214 #define CXL_AFU_Cntl_An_RS_MASK (0x3ull << (63-5))
215 #define CXL_AFU_Cntl_An_RS_Pending (0x1ull << (63-5))
216 #define CXL_AFU_Cntl_An_RS_Complete (0x2ull << (63-5))
217 /* AFU Slice Reset */
218 #define CXL_AFU_Cntl_An_RA (0x1ull << (63-7))
220 /****** CXL_SSTP0/1_An ******************************************************/
221 /* These top bits are for the segment that CONTAINS the segment table */
222 #define CXL_SSTP0_An_B_SHIFT SLB_VSID_SSIZE_SHIFT
223 #define CXL_SSTP0_An_KS (1ull << (63-2))
224 #define CXL_SSTP0_An_KP (1ull << (63-3))
225 #define CXL_SSTP0_An_N (1ull << (63-4))
226 #define CXL_SSTP0_An_L (1ull << (63-5))
227 #define CXL_SSTP0_An_C (1ull << (63-6))
228 #define CXL_SSTP0_An_TA (1ull << (63-7))
229 #define CXL_SSTP0_An_LP_SHIFT (63-9) /* 2 Bits */
230 /* And finally, the virtual address & size of the segment table: */
231 #define CXL_SSTP0_An_SegTableSize_SHIFT (63-31) /* 12 Bits */
232 #define CXL_SSTP0_An_SegTableSize_MASK \
233 (((1ull << 12) - 1) << CXL_SSTP0_An_SegTableSize_SHIFT)
234 #define CXL_SSTP0_An_STVA_U_MASK ((1ull << (63-49))-1)
235 #define CXL_SSTP1_An_STVA_L_MASK (~((1ull << (63-55))-1))
236 #define CXL_SSTP1_An_V (1ull << (63-63))
238 /****** CXL_PSL_SLBIE_[An] **************************************************/
240 #define CXL_SLBIE_C PPC_BIT(36) /* Class */
241 #define CXL_SLBIE_SS PPC_BITMASK(37, 38) /* Segment Size */
242 #define CXL_SLBIE_SS_SHIFT PPC_BITLSHIFT(38)
243 #define CXL_SLBIE_TA PPC_BIT(38) /* Tags Active */
245 #define CXL_SLBIE_MAX PPC_BITMASK(24, 31)
246 #define CXL_SLBIE_PENDING PPC_BITMASK(56, 63)
248 /****** Common to all CXL_TLBIA/SLBIA_[An] **********************************/
249 #define CXL_TLB_SLB_P (1ull) /* Pending (read) */
251 /****** Common to all CXL_TLB/SLB_IA/IE_[An] registers **********************/
252 #define CXL_TLB_SLB_IQ_ALL (0ull) /* Inv qualifier */
253 #define CXL_TLB_SLB_IQ_LPID (1ull) /* Inv qualifier */
254 #define CXL_TLB_SLB_IQ_LPIDPID (3ull) /* Inv qualifier */
256 /****** CXL_PSL_AFUSEL ******************************************************/
257 #define CXL_PSL_AFUSEL_A (1ull << (63-55)) /* Adapter wide invalidates affect all AFUs */
259 /****** CXL_PSL_DSISR_An ****************************************************/
260 #define CXL_PSL_DSISR_An_DS (1ull << (63-0)) /* Segment not found */
261 #define CXL_PSL_DSISR_An_DM (1ull << (63-1)) /* PTE not found (See also: M) or protection fault */
262 #define CXL_PSL_DSISR_An_ST (1ull << (63-2)) /* Segment Table PTE not found */
263 #define CXL_PSL_DSISR_An_UR (1ull << (63-3)) /* AURP PTE not found */
264 #define CXL_PSL_DSISR_TRANS (CXL_PSL_DSISR_An_DS | CXL_PSL_DSISR_An_DM | CXL_PSL_DSISR_An_ST | CXL_PSL_DSISR_An_UR)
265 #define CXL_PSL_DSISR_An_PE (1ull << (63-4)) /* PSL Error (implementation specific) */
266 #define CXL_PSL_DSISR_An_AE (1ull << (63-5)) /* AFU Error */
267 #define CXL_PSL_DSISR_An_OC (1ull << (63-6)) /* OS Context Warning */
268 #define CXL_PSL_DSISR_PENDING (CXL_PSL_DSISR_TRANS | CXL_PSL_DSISR_An_PE | CXL_PSL_DSISR_An_AE | CXL_PSL_DSISR_An_OC)
269 /* NOTE: Bits 32:63 are undefined if DSISR[DS] = 1 */
270 #define CXL_PSL_DSISR_An_M DSISR_NOHPTE /* PTE not found */
271 #define CXL_PSL_DSISR_An_P DSISR_PROTFAULT /* Storage protection violation */
272 #define CXL_PSL_DSISR_An_A (1ull << (63-37)) /* AFU lock access to write through or cache inhibited storage */
273 #define CXL_PSL_DSISR_An_S DSISR_ISSTORE /* Access was afu_wr or afu_zero */
274 #define CXL_PSL_DSISR_An_K DSISR_KEYFAULT /* Access not permitted by virtual page class key protection */
276 /****** CXL_PSL_TFC_An ******************************************************/
277 #define CXL_PSL_TFC_An_A (1ull << (63-28)) /* Acknowledge non-translation fault */
278 #define CXL_PSL_TFC_An_C (1ull << (63-29)) /* Continue (abort transaction) */
279 #define CXL_PSL_TFC_An_AE (1ull << (63-30)) /* Restart PSL with address error */
280 #define CXL_PSL_TFC_An_R (1ull << (63-31)) /* Restart PSL transaction */
282 /* cxl_process_element->software_status */
283 #define CXL_PE_SOFTWARE_STATE_V (1ul << (31 - 0)) /* Valid */
284 #define CXL_PE_SOFTWARE_STATE_C (1ul << (31 - 29)) /* Complete */
285 #define CXL_PE_SOFTWARE_STATE_S (1ul << (31 - 30)) /* Suspend */
286 #define CXL_PE_SOFTWARE_STATE_T (1ul << (31 - 31)) /* Terminate */
288 /****** CXL_PSL_RXCTL_An (Implementation Specific) **************************
289 * Controls AFU Hang Pulse, which sets the timeout for the AFU to respond to
290 * the PSL for any response (except MMIO). Timeouts will occur between 1x to 2x
291 * of the hang pulse frequency.
293 #define CXL_PSL_RXCTL_AFUHP_4S 0x7000000000000000ULL
295 /* SPA->sw_command_status */
296 #define CXL_SPA_SW_CMD_MASK 0xffff000000000000ULL
297 #define CXL_SPA_SW_CMD_TERMINATE 0x0001000000000000ULL
298 #define CXL_SPA_SW_CMD_REMOVE 0x0002000000000000ULL
299 #define CXL_SPA_SW_CMD_SUSPEND 0x0003000000000000ULL
300 #define CXL_SPA_SW_CMD_RESUME 0x0004000000000000ULL
301 #define CXL_SPA_SW_CMD_ADD 0x0005000000000000ULL
302 #define CXL_SPA_SW_CMD_UPDATE 0x0006000000000000ULL
303 #define CXL_SPA_SW_STATE_MASK 0x0000ffff00000000ULL
304 #define CXL_SPA_SW_STATE_TERMINATED 0x0000000100000000ULL
305 #define CXL_SPA_SW_STATE_REMOVED 0x0000000200000000ULL
306 #define CXL_SPA_SW_STATE_SUSPENDED 0x0000000300000000ULL
307 #define CXL_SPA_SW_STATE_RESUMED 0x0000000400000000ULL
308 #define CXL_SPA_SW_STATE_ADDED 0x0000000500000000ULL
309 #define CXL_SPA_SW_STATE_UPDATED 0x0000000600000000ULL
310 #define CXL_SPA_SW_PSL_ID_MASK 0x00000000ffff0000ULL
311 #define CXL_SPA_SW_LINK_MASK 0x000000000000ffffULL
313 #define CXL_MAX_SLICES 4
314 #define MAX_AFU_MMIO_REGS 3
316 #define CXL_MODE_TIME_SLICED 0x4
317 #define CXL_SUPPORTED_MODES (CXL_MODE_DEDICATED | CXL_MODE_DIRECTED)
319 #define CXL_DEV_MINORS 13 /* 1 control + 4 AFUs * 3 (dedicated/master/shared) */
320 #define CXL_CARD_MINOR(adapter) (adapter->adapter_num * CXL_DEV_MINORS)
321 #define CXL_DEVT_ADAPTER(dev) (MINOR(dev) / CXL_DEV_MINORS)
323 enum cxl_context_status
{
329 enum prefault_modes
{
337 CXL_AFU_MASTER_ATTRS
,
346 #define to_cxl_adapter(d) container_of(d, struct cxl, dev)
347 #define to_cxl_afu(d) container_of(d, struct cxl_afu, dev)
349 struct cxl_afu_native
{
350 void __iomem
*p1n_mmio
;
351 void __iomem
*afu_desc_mmio
;
352 irq_hw_number_t psl_hwirq
;
353 unsigned int psl_virq
;
354 struct mutex spa_mutex
;
356 * Only the first part of the SPA is used for the process element
357 * linked list. The only other part that software needs to worry about
358 * is sw_command_status, which we store a separate pointer to.
359 * Everything else in the SPA is only used by hardware
361 struct cxl_process_element
*spa
;
362 __be64
*sw_command_status
;
363 unsigned int spa_size
;
369 struct cxl_afu_guest
{
370 struct cxl_afu
*parent
;
372 phys_addr_t p2n_phys
;
376 struct delayed_work work_err
;
381 struct cxl_afu_native
*native
;
382 struct cxl_afu_guest
*guest
;
383 irq_hw_number_t serr_hwirq
;
384 unsigned int serr_virq
;
387 void __iomem
*p2n_mmio
;
388 phys_addr_t psn_phys
;
393 struct cdev afu_cdev_s
, afu_cdev_m
, afu_cdev_d
;
394 struct device
*chardev_s
, *chardev_m
, *chardev_d
;
395 struct idr contexts_idr
;
396 struct dentry
*debugfs
;
397 struct mutex contexts_lock
;
398 spinlock_t afu_cntl_lock
;
400 /* AFU error buffer fields and bin attribute for sysfs */
401 u64 eb_len
, eb_offset
;
402 struct bin_attribute attr_eb
;
404 /* pointer to the vphb */
405 struct pci_controller
*phb
;
410 int max_procs_virtualised
;
417 struct list_head crs
;
418 enum prefault_modes prefault_mode
;
424 /* AFU refcount management */
425 static inline struct cxl_afu
*cxl_afu_get(struct cxl_afu
*afu
)
428 return (get_device(&afu
->dev
) == NULL
) ? NULL
: afu
;
431 static inline void cxl_afu_put(struct cxl_afu
*afu
)
433 put_device(&afu
->dev
);
437 struct cxl_irq_name
{
438 struct list_head list
;
443 irq_hw_number_t offset
;
444 irq_hw_number_t range
;
445 unsigned long *bitmap
;
449 * This is a cxl context. If the PSL is in dedicated mode, there will be one
450 * of these per AFU. If in AFU directed there can be lots of these.
455 /* Problem state MMIO */
456 phys_addr_t psn_phys
;
459 /* Used to unmap any mmaps when force detaching */
460 struct address_space
*mapping
;
461 struct mutex mapping_lock
;
462 struct page
*ff_page
;
466 spinlock_t sste_lock
; /* Protects segment table entries */
467 struct cxl_sste
*sstp
;
469 unsigned int sst_size
, sst_lru
;
471 wait_queue_head_t wq
;
472 /* pid of the group leader associated with the pid */
474 /* use mm context associated with this pid for ds faults */
476 spinlock_t lock
; /* Protects pending_irq_mask, pending_fault and fault_addr */
477 /* Only used in PR mode */
480 unsigned long *irq_bitmap
; /* Accessed from IRQ context */
481 struct cxl_irq_ranges irqs
;
482 struct list_head irq_names
;
488 * This status and it's lock pretects start and detach context
489 * from racing. It also prevents detach from racing with
492 enum cxl_context_status status
;
493 struct mutex status_mutex
;
496 /* XXX: Is it possible to need multiple work items at once? */
497 struct work_struct fault_work
;
501 struct cxl_process_element
*elem
;
504 * pe is the process element handle, assigned by this driver when the
505 * context is initialized.
507 * external_pe is the PE shown outside of cxl.
508 * On bare-metal, pe=external_pe, because we decide what the handle is.
509 * In a guest, we only find out about the pe used by pHyp when the
510 * context is attached, and that's the value we want to report outside
523 bool pending_afu_err
;
531 void __iomem
*p1_mmio
;
532 void __iomem
*p2_mmio
;
533 irq_hw_number_t err_hwirq
;
534 unsigned int err_virq
;
539 struct platform_device
*pdev
;
542 irq_hw_number_t irq_base_offset
;
543 struct irq_avail
*irq_avail
;
544 spinlock_t irq_alloc_lock
;
549 u16 subsystem_vendor
;
554 struct cxl_native
*native
;
555 struct cxl_guest
*guest
;
556 spinlock_t afu_list_lock
;
557 struct cxl_afu
*afu
[CXL_MAX_SLICES
];
559 struct dentry
*trace
;
560 struct dentry
*psl_err_chk
;
561 struct dentry
*debugfs
;
563 struct bin_attribute cxl_attr
;
573 bool user_image_loaded
;
574 bool perst_loads_image
;
575 bool perst_select_user
;
576 bool perst_same_image
;
577 bool psl_timebase_synced
;
580 int cxl_pci_alloc_one_irq(struct cxl
*adapter
);
581 void cxl_pci_release_one_irq(struct cxl
*adapter
, int hwirq
);
582 int cxl_pci_alloc_irq_ranges(struct cxl_irq_ranges
*irqs
, struct cxl
*adapter
, unsigned int num
);
583 void cxl_pci_release_irq_ranges(struct cxl_irq_ranges
*irqs
, struct cxl
*adapter
);
584 int cxl_pci_setup_irq(struct cxl
*adapter
, unsigned int hwirq
, unsigned int virq
);
585 int cxl_update_image_control(struct cxl
*adapter
);
586 int cxl_pci_reset(struct cxl
*adapter
);
587 void cxl_pci_release_afu(struct device
*dev
);
588 ssize_t
cxl_pci_read_adapter_vpd(struct cxl
*adapter
, void *buf
, size_t len
);
590 /* common == phyp + powernv */
591 struct cxl_process_element_common
{
605 struct cxl_process_element
{
611 __be16 ivte_offsets
[4];
612 __be16 ivte_ranges
[4];
614 struct cxl_process_element_common common
;
615 __be32 software_state
;
618 static inline bool cxl_adapter_link_ok(struct cxl
*cxl
, struct cxl_afu
*afu
)
620 struct pci_dev
*pdev
;
622 if (cpu_has_feature(CPU_FTR_HVMODE
)) {
623 pdev
= to_pci_dev(cxl
->dev
.parent
);
624 return !pci_channel_offline(pdev
);
629 static inline void __iomem
*_cxl_p1_addr(struct cxl
*cxl
, cxl_p1_reg_t reg
)
631 WARN_ON(!cpu_has_feature(CPU_FTR_HVMODE
));
632 return cxl
->native
->p1_mmio
+ cxl_reg_off(reg
);
635 static inline void cxl_p1_write(struct cxl
*cxl
, cxl_p1_reg_t reg
, u64 val
)
637 if (likely(cxl_adapter_link_ok(cxl
, NULL
)))
638 out_be64(_cxl_p1_addr(cxl
, reg
), val
);
641 static inline u64
cxl_p1_read(struct cxl
*cxl
, cxl_p1_reg_t reg
)
643 if (likely(cxl_adapter_link_ok(cxl
, NULL
)))
644 return in_be64(_cxl_p1_addr(cxl
, reg
));
649 static inline void __iomem
*_cxl_p1n_addr(struct cxl_afu
*afu
, cxl_p1n_reg_t reg
)
651 WARN_ON(!cpu_has_feature(CPU_FTR_HVMODE
));
652 return afu
->native
->p1n_mmio
+ cxl_reg_off(reg
);
655 static inline void cxl_p1n_write(struct cxl_afu
*afu
, cxl_p1n_reg_t reg
, u64 val
)
657 if (likely(cxl_adapter_link_ok(afu
->adapter
, afu
)))
658 out_be64(_cxl_p1n_addr(afu
, reg
), val
);
661 static inline u64
cxl_p1n_read(struct cxl_afu
*afu
, cxl_p1n_reg_t reg
)
663 if (likely(cxl_adapter_link_ok(afu
->adapter
, afu
)))
664 return in_be64(_cxl_p1n_addr(afu
, reg
));
669 static inline void __iomem
*_cxl_p2n_addr(struct cxl_afu
*afu
, cxl_p2n_reg_t reg
)
671 return afu
->p2n_mmio
+ cxl_reg_off(reg
);
674 static inline void cxl_p2n_write(struct cxl_afu
*afu
, cxl_p2n_reg_t reg
, u64 val
)
676 if (likely(cxl_adapter_link_ok(afu
->adapter
, afu
)))
677 out_be64(_cxl_p2n_addr(afu
, reg
), val
);
680 static inline u64
cxl_p2n_read(struct cxl_afu
*afu
, cxl_p2n_reg_t reg
)
682 if (likely(cxl_adapter_link_ok(afu
->adapter
, afu
)))
683 return in_be64(_cxl_p2n_addr(afu
, reg
));
688 ssize_t
cxl_pci_afu_read_err_buffer(struct cxl_afu
*afu
, char *buf
,
689 loff_t off
, size_t count
);
693 void (*cxl_slbia
)(struct mm_struct
*mm
);
694 struct module
*owner
;
696 int register_cxl_calls(struct cxl_calls
*calls
);
697 void unregister_cxl_calls(struct cxl_calls
*calls
);
698 int cxl_update_properties(struct device_node
*dn
, struct property
*new_prop
);
700 void cxl_remove_adapter_nr(struct cxl
*adapter
);
702 int cxl_alloc_spa(struct cxl_afu
*afu
);
703 void cxl_release_spa(struct cxl_afu
*afu
);
705 dev_t
cxl_get_dev(void);
706 int cxl_file_init(void);
707 void cxl_file_exit(void);
708 int cxl_register_adapter(struct cxl
*adapter
);
709 int cxl_register_afu(struct cxl_afu
*afu
);
710 int cxl_chardev_d_afu_add(struct cxl_afu
*afu
);
711 int cxl_chardev_m_afu_add(struct cxl_afu
*afu
);
712 int cxl_chardev_s_afu_add(struct cxl_afu
*afu
);
713 void cxl_chardev_afu_remove(struct cxl_afu
*afu
);
715 void cxl_context_detach_all(struct cxl_afu
*afu
);
716 void cxl_context_free(struct cxl_context
*ctx
);
717 void cxl_context_detach(struct cxl_context
*ctx
);
719 int cxl_sysfs_adapter_add(struct cxl
*adapter
);
720 void cxl_sysfs_adapter_remove(struct cxl
*adapter
);
721 int cxl_sysfs_afu_add(struct cxl_afu
*afu
);
722 void cxl_sysfs_afu_remove(struct cxl_afu
*afu
);
723 int cxl_sysfs_afu_m_add(struct cxl_afu
*afu
);
724 void cxl_sysfs_afu_m_remove(struct cxl_afu
*afu
);
726 struct cxl
*cxl_alloc_adapter(void);
727 struct cxl_afu
*cxl_alloc_afu(struct cxl
*adapter
, int slice
);
728 int cxl_afu_select_best_mode(struct cxl_afu
*afu
);
730 int cxl_native_register_psl_irq(struct cxl_afu
*afu
);
731 void cxl_native_release_psl_irq(struct cxl_afu
*afu
);
732 int cxl_native_register_psl_err_irq(struct cxl
*adapter
);
733 void cxl_native_release_psl_err_irq(struct cxl
*adapter
);
734 int cxl_native_register_serr_irq(struct cxl_afu
*afu
);
735 void cxl_native_release_serr_irq(struct cxl_afu
*afu
);
736 int afu_register_irqs(struct cxl_context
*ctx
, u32 count
);
737 void afu_release_irqs(struct cxl_context
*ctx
, void *cookie
);
738 void afu_irq_name_free(struct cxl_context
*ctx
);
740 int cxl_debugfs_init(void);
741 void cxl_debugfs_exit(void);
742 int cxl_debugfs_adapter_add(struct cxl
*adapter
);
743 void cxl_debugfs_adapter_remove(struct cxl
*adapter
);
744 int cxl_debugfs_afu_add(struct cxl_afu
*afu
);
745 void cxl_debugfs_afu_remove(struct cxl_afu
*afu
);
747 void cxl_handle_fault(struct work_struct
*work
);
748 void cxl_prefault(struct cxl_context
*ctx
, u64 wed
);
750 struct cxl
*get_cxl_adapter(int num
);
751 int cxl_alloc_sst(struct cxl_context
*ctx
);
752 void cxl_dump_debug_buffer(void *addr
, size_t size
);
754 void init_cxl_native(void);
756 struct cxl_context
*cxl_context_alloc(void);
757 int cxl_context_init(struct cxl_context
*ctx
, struct cxl_afu
*afu
, bool master
,
758 struct address_space
*mapping
);
759 void cxl_context_free(struct cxl_context
*ctx
);
760 int cxl_context_iomap(struct cxl_context
*ctx
, struct vm_area_struct
*vma
);
761 unsigned int cxl_map_irq(struct cxl
*adapter
, irq_hw_number_t hwirq
,
762 irq_handler_t handler
, void *cookie
, const char *name
);
763 void cxl_unmap_irq(unsigned int virq
, void *cookie
);
764 int __detach_context(struct cxl_context
*ctx
);
767 * This must match the layout of the H_COLLECT_CA_INT_INFO retbuf defined
769 * A word about endianness: a pointer to this structure is passed when
770 * calling the hcall. However, it is not a block of memory filled up by
771 * the hypervisor. The return values are found in registers, and copied
772 * one by one when returning from the hcall. See the end of the call to
773 * plpar_hcall9() in hvCall.S
775 * - we don't need to do any endianness conversion
776 * - the pid and tid are an exception. They are 32-bit values returned in
777 * the same 64-bit register. So we do need to worry about byte ordering.
779 struct cxl_irq_info
{
783 #ifndef CONFIG_CPU_LITTLE_ENDIAN
793 u64 padding
[2]; /* to match the expected retbuf size for plpar_hcall9 */
796 void cxl_assign_psn_space(struct cxl_context
*ctx
);
797 irqreturn_t
cxl_irq(int irq
, struct cxl_context
*ctx
, struct cxl_irq_info
*irq_info
);
798 int cxl_register_one_irq(struct cxl
*adapter
, irq_handler_t handler
,
799 void *cookie
, irq_hw_number_t
*dest_hwirq
,
800 unsigned int *dest_virq
, const char *name
);
802 int cxl_check_error(struct cxl_afu
*afu
);
803 int cxl_afu_slbia(struct cxl_afu
*afu
);
804 int cxl_tlb_slb_invalidate(struct cxl
*adapter
);
805 int cxl_afu_disable(struct cxl_afu
*afu
);
806 int cxl_psl_purge(struct cxl_afu
*afu
);
808 void cxl_stop_trace(struct cxl
*cxl
);
809 int cxl_pci_vphb_add(struct cxl_afu
*afu
);
810 void cxl_pci_vphb_remove(struct cxl_afu
*afu
);
812 extern struct pci_driver cxl_pci_driver
;
813 extern struct platform_driver cxl_of_driver
;
814 int afu_allocate_irqs(struct cxl_context
*ctx
, u32 count
);
816 int afu_open(struct inode
*inode
, struct file
*file
);
817 int afu_release(struct inode
*inode
, struct file
*file
);
818 long afu_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
);
819 int afu_mmap(struct file
*file
, struct vm_area_struct
*vm
);
820 unsigned int afu_poll(struct file
*file
, struct poll_table_struct
*poll
);
821 ssize_t
afu_read(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*off
);
822 extern const struct file_operations afu_fops
;
824 struct cxl
*cxl_guest_init_adapter(struct device_node
*np
, struct platform_device
*dev
);
825 void cxl_guest_remove_adapter(struct cxl
*adapter
);
826 int cxl_of_read_adapter_handle(struct cxl
*adapter
, struct device_node
*np
);
827 int cxl_of_read_adapter_properties(struct cxl
*adapter
, struct device_node
*np
);
828 ssize_t
cxl_guest_read_adapter_vpd(struct cxl
*adapter
, void *buf
, size_t len
);
829 ssize_t
cxl_guest_read_afu_vpd(struct cxl_afu
*afu
, void *buf
, size_t len
);
830 int cxl_guest_init_afu(struct cxl
*adapter
, int slice
, struct device_node
*afu_np
);
831 void cxl_guest_remove_afu(struct cxl_afu
*afu
);
832 int cxl_of_read_afu_handle(struct cxl_afu
*afu
, struct device_node
*afu_np
);
833 int cxl_of_read_afu_properties(struct cxl_afu
*afu
, struct device_node
*afu_np
);
834 int cxl_guest_add_chardev(struct cxl
*adapter
);
835 void cxl_guest_remove_chardev(struct cxl
*adapter
);
836 void cxl_guest_reload_module(struct cxl
*adapter
);
837 int cxl_of_probe(struct platform_device
*pdev
);
839 struct cxl_backend_ops
{
840 struct module
*module
;
841 int (*adapter_reset
)(struct cxl
*adapter
);
842 int (*alloc_one_irq
)(struct cxl
*adapter
);
843 void (*release_one_irq
)(struct cxl
*adapter
, int hwirq
);
844 int (*alloc_irq_ranges
)(struct cxl_irq_ranges
*irqs
,
845 struct cxl
*adapter
, unsigned int num
);
846 void (*release_irq_ranges
)(struct cxl_irq_ranges
*irqs
,
847 struct cxl
*adapter
);
848 int (*setup_irq
)(struct cxl
*adapter
, unsigned int hwirq
,
850 irqreturn_t (*handle_psl_slice_error
)(struct cxl_context
*ctx
,
851 u64 dsisr
, u64 errstat
);
852 irqreturn_t (*psl_interrupt
)(int irq
, void *data
);
853 int (*ack_irq
)(struct cxl_context
*ctx
, u64 tfc
, u64 psl_reset_mask
);
854 void (*irq_wait
)(struct cxl_context
*ctx
);
855 int (*attach_process
)(struct cxl_context
*ctx
, bool kernel
,
857 int (*detach_process
)(struct cxl_context
*ctx
);
858 bool (*support_attributes
)(const char *attr_name
, enum cxl_attrs type
);
859 bool (*link_ok
)(struct cxl
*cxl
, struct cxl_afu
*afu
);
860 void (*release_afu
)(struct device
*dev
);
861 ssize_t (*afu_read_err_buffer
)(struct cxl_afu
*afu
, char *buf
,
862 loff_t off
, size_t count
);
863 int (*afu_check_and_enable
)(struct cxl_afu
*afu
);
864 int (*afu_activate_mode
)(struct cxl_afu
*afu
, int mode
);
865 int (*afu_deactivate_mode
)(struct cxl_afu
*afu
, int mode
);
866 int (*afu_reset
)(struct cxl_afu
*afu
);
867 int (*afu_cr_read8
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u8
*val
);
868 int (*afu_cr_read16
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u16
*val
);
869 int (*afu_cr_read32
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u32
*val
);
870 int (*afu_cr_read64
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u64
*val
);
871 int (*afu_cr_write8
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u8 val
);
872 int (*afu_cr_write16
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u16 val
);
873 int (*afu_cr_write32
)(struct cxl_afu
*afu
, int cr_idx
, u64 offset
, u32 val
);
874 ssize_t (*read_adapter_vpd
)(struct cxl
*adapter
, void *buf
, size_t count
);
876 extern const struct cxl_backend_ops cxl_native_ops
;
877 extern const struct cxl_backend_ops cxl_guest_ops
;
878 extern const struct cxl_backend_ops
*cxl_ops
;
880 /* check if the given pci_dev is on the the cxl vphb bus */
881 bool cxl_pci_is_vphb_device(struct pci_dev
*dev
);