projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
staging/rdma/hfi1: fix 0-day syntax error
[deliverable/linux.git] / drivers / staging / rdma / hfi1 / chip.c
CommitLineData
77241056 1/*
05d6ac1d 2 * Copyright(c) 2015, 2016 Intel Corporation.
77241056
MM		 3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
77241056
MM		 9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
77241056
MM		 20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48/*
49 * This file contains all of the code that is specific to the HFI chip
50 */
51
52#include <linux/pci.h>
53#include <linux/delay.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56
57#include "hfi.h"
58#include "trace.h"
59#include "mad.h"
60#include "pio.h"
61#include "sdma.h"
62#include "eprom.h"
5d9157aa 63#include "efivar.h"
8ebd4cf1 64#include "platform.h"
affa48de 65#include "aspm.h"
77241056
MM		 66
67#define NUM_IB_PORTS 1
68
69uint kdeth_qp;
70module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
71MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
72
73uint num_vls = HFI1_MAX_VLS_SUPPORTED;
74module_param(num_vls, uint, S_IRUGO);
75MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
76
77/*
78 * Default time to aggregate two 10K packets from the idle state
79 * (timer not running). The timer starts at the end of the first packet,
80 * so only the time for one 10K packet and header plus a bit extra is needed.
81 * 10 * 1024 + 64 header byte = 10304 byte
82 * 10304 byte / 12.5 GB/s = 824.32ns
83 */
84uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
85module_param(rcv_intr_timeout, uint, S_IRUGO);
86MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
87
88uint rcv_intr_count = 16; /* same as qib */
89module_param(rcv_intr_count, uint, S_IRUGO);
90MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
91
92ushort link_crc_mask = SUPPORTED_CRCS;
93module_param(link_crc_mask, ushort, S_IRUGO);
94MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
95
96uint loopback;
97module_param_named(loopback, loopback, uint, S_IRUGO);
98MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
99
100/* Other driver tunables */
101uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
102static ushort crc_14b_sideband = 1;
103static uint use_flr = 1;
104uint quick_linkup; /* skip LNI */
105
106struct flag_table {
107 u64 flag; /* the flag */
108 char *str; /* description string */
109 u16 extra; /* extra information */
110 u16 unused0;
111 u32 unused1;
112};
113
114/* str must be a string constant */
115#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
116#define FLAG_ENTRY0(str, flag) {flag, str, 0}
117
118/* Send Error Consequences */
119#define SEC_WRITE_DROPPED 0x1
120#define SEC_PACKET_DROPPED 0x2
121#define SEC_SC_HALTED 0x4 /* per-context only */
122#define SEC_SPC_FREEZE 0x8 /* per-HFI only */
123
77241056 124#define MIN_KERNEL_KCTXTS 2
82c2611d 125#define FIRST_KERNEL_KCTXT 1
77241056
MM		 126#define NUM_MAP_REGS 32
127
128/* Bit offset into the GUID which carries HFI id information */
129#define GUID_HFI_INDEX_SHIFT 39
130
131/* extract the emulation revision */
132#define emulator_rev(dd) ((dd)->irev >> 8)
133/* parallel and serial emulation versions are 3 and 4 respectively */
134#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
135#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
136
137/* RSM fields */
138
139/* packet type */
140#define IB_PACKET_TYPE 2ull
141#define QW_SHIFT 6ull
142/* QPN[7..1] */
143#define QPN_WIDTH 7ull
144
145/* LRH.BTH: QW 0, OFFSET 48 - for match */
146#define LRH_BTH_QW 0ull
147#define LRH_BTH_BIT_OFFSET 48ull
148#define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
149#define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
150#define LRH_BTH_SELECT
151#define LRH_BTH_MASK 3ull
152#define LRH_BTH_VALUE 2ull
153
154/* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
155#define LRH_SC_QW 0ull
156#define LRH_SC_BIT_OFFSET 56ull
157#define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
158#define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
159#define LRH_SC_MASK 128ull
160#define LRH_SC_VALUE 0ull
161
162/* SC[n..0] QW 0, OFFSET 60 - for select */
163#define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
164
165/* QPN[m+n:1] QW 1, OFFSET 1 */
166#define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
167
168/* defines to build power on SC2VL table */
169#define SC2VL_VAL( \
170 num, \
171 sc0, sc0val, \
172 sc1, sc1val, \
173 sc2, sc2val, \
174 sc3, sc3val, \
175 sc4, sc4val, \
176 sc5, sc5val, \
177 sc6, sc6val, \
178 sc7, sc7val) \
179( \
180 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
181 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
182 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
183 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
184 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
185 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
186 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
187 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
188)
189
190#define DC_SC_VL_VAL( \
191 range, \
192 e0, e0val, \
193 e1, e1val, \
194 e2, e2val, \
195 e3, e3val, \
196 e4, e4val, \
197 e5, e5val, \
198 e6, e6val, \
199 e7, e7val, \
200 e8, e8val, \
201 e9, e9val, \
202 e10, e10val, \
203 e11, e11val, \
204 e12, e12val, \
205 e13, e13val, \
206 e14, e14val, \
207 e15, e15val) \
208( \
209 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
210 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
211 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
212 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
213 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
214 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
215 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
216 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
217 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
218 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
219 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
220 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
221 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
222 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
223 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
224 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
225)
226
227/* all CceStatus sub-block freeze bits */
228#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
229 | CCE_STATUS_RXE_FROZE_SMASK \
230 | CCE_STATUS_TXE_FROZE_SMASK \
231 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
232/* all CceStatus sub-block TXE pause bits */
233#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
234 | CCE_STATUS_TXE_PAUSED_SMASK \
235 | CCE_STATUS_SDMA_PAUSED_SMASK)
236/* all CceStatus sub-block RXE pause bits */
237#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
238
239/*
240 * CCE Error flags.
241 */
242static struct flag_table cce_err_status_flags[] = {
243/* 0*/ FLAG_ENTRY0("CceCsrParityErr",
244 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
245/* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
246 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
247/* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
248 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
249/* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
250 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
251/* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
252 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
253/* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
254 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
255/* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
256 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
257/* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
258 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
259/* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
260 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
261/* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
262 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
263/*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
264 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
265/*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
266 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
267/*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
268 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
269/*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
270 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
271/*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
272 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
273/*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
274 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
275/*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
276 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
277/*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
278 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
279/*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
280 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
281/*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
282 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
283/*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
284 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
285/*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
286 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
287/*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
288 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
289/*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
290 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
291/*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
292 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
293/*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
294 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
295/*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
296 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
297/*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
298 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
299/*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
300 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
301/*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
302 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
303/*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
304 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
305/*31*/ FLAG_ENTRY0("LATriggered",
306 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
307/*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
308 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
309/*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
310 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
311/*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
312 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
313/*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
314 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
315/*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
316 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
317/*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
318 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
319/*38*/ FLAG_ENTRY0("CceIntMapCorErr",
320 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
321/*39*/ FLAG_ENTRY0("CceIntMapUncErr",
322 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
323/*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
324 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
325/*41-63 reserved*/
326};
327
328/*
329 * Misc Error flags
330 */
331#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
332static struct flag_table misc_err_status_flags[] = {
333/* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
334/* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
335/* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
336/* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
337/* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
338/* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
339/* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
340/* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
341/* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
342/* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
343/*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
344/*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
345/*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
346};
347
348/*
349 * TXE PIO Error flags and consequences
350 */
351static struct flag_table pio_err_status_flags[] = {
352/* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
353 SEC_WRITE_DROPPED,
354 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
355/* 1*/ FLAG_ENTRY("PioWriteAddrParity",
356 SEC_SPC_FREEZE,
357 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
358/* 2*/ FLAG_ENTRY("PioCsrParity",
359 SEC_SPC_FREEZE,
360 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
361/* 3*/ FLAG_ENTRY("PioSbMemFifo0",
362 SEC_SPC_FREEZE,
363 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
364/* 4*/ FLAG_ENTRY("PioSbMemFifo1",
365 SEC_SPC_FREEZE,
366 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
367/* 5*/ FLAG_ENTRY("PioPccFifoParity",
368 SEC_SPC_FREEZE,
369 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
370/* 6*/ FLAG_ENTRY("PioPecFifoParity",
371 SEC_SPC_FREEZE,
372 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
373/* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
374 SEC_SPC_FREEZE,
375 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
376/* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
377 SEC_SPC_FREEZE,
378 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
379/* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
380 SEC_SPC_FREEZE,
381 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
382/*10*/ FLAG_ENTRY("PioSmPktResetParity",
383 SEC_SPC_FREEZE,
384 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
385/*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
386 SEC_SPC_FREEZE,
387 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
388/*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
389 SEC_SPC_FREEZE,
390 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
391/*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
392 0,
393 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
394/*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
395 0,
396 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
397/*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
398 SEC_SPC_FREEZE,
399 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
400/*16*/ FLAG_ENTRY("PioPpmcPblFifo",
401 SEC_SPC_FREEZE,
402 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
403/*17*/ FLAG_ENTRY("PioInitSmIn",
404 0,
405 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
406/*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
407 SEC_SPC_FREEZE,
408 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
409/*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
410 SEC_SPC_FREEZE,
411 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
412/*20*/ FLAG_ENTRY("PioHostAddrMemCor",
413 0,
414 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
415/*21*/ FLAG_ENTRY("PioWriteDataParity",
416 SEC_SPC_FREEZE,
417 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
418/*22*/ FLAG_ENTRY("PioStateMachine",
419 SEC_SPC_FREEZE,
420 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
421/*23*/ FLAG_ENTRY("PioWriteQwValidParity",
8638b77f 422 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
77241056
MM		 423 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
424/*24*/ FLAG_ENTRY("PioBlockQwCountParity",
8638b77f 425 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
77241056
MM		 426 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
427/*25*/ FLAG_ENTRY("PioVlfVlLenParity",
428 SEC_SPC_FREEZE,
429 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
430/*26*/ FLAG_ENTRY("PioVlfSopParity",
431 SEC_SPC_FREEZE,
432 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
433/*27*/ FLAG_ENTRY("PioVlFifoParity",
434 SEC_SPC_FREEZE,
435 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
436/*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
437 SEC_SPC_FREEZE,
438 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
439/*29*/ FLAG_ENTRY("PioPpmcSopLen",
440 SEC_SPC_FREEZE,
441 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
442/*30-31 reserved*/
443/*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
444 SEC_SPC_FREEZE,
445 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
446/*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
447 SEC_SPC_FREEZE,
448 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
449/*34*/ FLAG_ENTRY("PioPccSopHeadParity",
450 SEC_SPC_FREEZE,
451 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
452/*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
453 SEC_SPC_FREEZE,
454 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
455/*36-63 reserved*/
456};
457
458/* TXE PIO errors that cause an SPC freeze */
459#define ALL_PIO_FREEZE_ERR \
460 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
461 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
462 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
463 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
464 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
465 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
466 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
467 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
468 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
469 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
470 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
471 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
489
490/*
491 * TXE SDMA Error flags
492 */
493static struct flag_table sdma_err_status_flags[] = {
494/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
495 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
496/* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
497 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
498/* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
499 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
500/* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
501 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
502/*04-63 reserved*/
503};
504
505/* TXE SDMA errors that cause an SPC freeze */
506#define ALL_SDMA_FREEZE_ERR \
507 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
508 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
509 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
510
69a00b8e
MM		 511/* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
512#define PORT_DISCARD_EGRESS_ERRS \
513 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
514 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
515 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
516
77241056
MM		 517/*
518 * TXE Egress Error flags
519 */
520#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
521static struct flag_table egress_err_status_flags[] = {
522/* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
523/* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
524/* 2 reserved */
525/* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
526 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
527/* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
528/* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
529/* 6 reserved */
530/* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
531 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
532/* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
533 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
534/* 9-10 reserved */
535/*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
536 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
537/*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
538/*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
539/*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
540/*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
541/*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
542 SEES(TX_SDMA0_DISALLOWED_PACKET)),
543/*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
544 SEES(TX_SDMA1_DISALLOWED_PACKET)),
545/*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
546 SEES(TX_SDMA2_DISALLOWED_PACKET)),
547/*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
548 SEES(TX_SDMA3_DISALLOWED_PACKET)),
549/*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
550 SEES(TX_SDMA4_DISALLOWED_PACKET)),
551/*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
552 SEES(TX_SDMA5_DISALLOWED_PACKET)),
553/*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
554 SEES(TX_SDMA6_DISALLOWED_PACKET)),
555/*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
556 SEES(TX_SDMA7_DISALLOWED_PACKET)),
557/*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
558 SEES(TX_SDMA8_DISALLOWED_PACKET)),
559/*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
560 SEES(TX_SDMA9_DISALLOWED_PACKET)),
561/*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
562 SEES(TX_SDMA10_DISALLOWED_PACKET)),
563/*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
564 SEES(TX_SDMA11_DISALLOWED_PACKET)),
565/*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
566 SEES(TX_SDMA12_DISALLOWED_PACKET)),
567/*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
568 SEES(TX_SDMA13_DISALLOWED_PACKET)),
569/*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
570 SEES(TX_SDMA14_DISALLOWED_PACKET)),
571/*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
572 SEES(TX_SDMA15_DISALLOWED_PACKET)),
573/*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
574 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
575/*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
576 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
577/*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
578 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
579/*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
580 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
581/*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
582 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
583/*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
584 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
585/*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
586 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
587/*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
588 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
589/*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
590 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
591/*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
592/*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
593/*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
594/*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
595/*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
596/*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
597/*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
598/*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
599/*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
600/*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
601/*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
602/*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
603/*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
604/*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
605/*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
606/*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
607/*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
608/*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
609/*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
610/*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
611/*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
612/*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
613 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
614/*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
615 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
616};
617
618/*
619 * TXE Egress Error Info flags
620 */
621#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
622static struct flag_table egress_err_info_flags[] = {
623/* 0*/ FLAG_ENTRY0("Reserved", 0ull),
624/* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
625/* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
626/* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
627/* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
628/* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
629/* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
630/* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
631/* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
632/* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
633/*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
634/*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
635/*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
636/*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
637/*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
638/*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
639/*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
640/*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
641/*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
642/*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
643/*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
644/*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
645};
646
647/* TXE Egress errors that cause an SPC freeze */
648#define ALL_TXE_EGRESS_FREEZE_ERR \
649 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
650 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
651 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
652 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
653 | SEES(TX_LAUNCH_CSR_PARITY) \
654 | SEES(TX_SBRD_CTL_CSR_PARITY) \
655 | SEES(TX_CONFIG_PARITY) \
656 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
657 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
658 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
659 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
660 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
661 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
662 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
663 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
664 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
665 | SEES(TX_CREDIT_RETURN_PARITY))
666
667/*
668 * TXE Send error flags
669 */
670#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
671static struct flag_table send_err_status_flags[] = {
2c5b521a 672/* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
77241056
MM		 673/* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
674/* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
675};
676
677/*
678 * TXE Send Context Error flags and consequences
679 */
680static struct flag_table sc_err_status_flags[] = {
681/* 0*/ FLAG_ENTRY("InconsistentSop",
682 SEC_PACKET_DROPPED | SEC_SC_HALTED,
683 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
684/* 1*/ FLAG_ENTRY("DisallowedPacket",
685 SEC_PACKET_DROPPED | SEC_SC_HALTED,
686 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
687/* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
688 SEC_WRITE_DROPPED | SEC_SC_HALTED,
689 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
690/* 3*/ FLAG_ENTRY("WriteOverflow",
691 SEC_WRITE_DROPPED | SEC_SC_HALTED,
692 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
693/* 4*/ FLAG_ENTRY("WriteOutOfBounds",
694 SEC_WRITE_DROPPED | SEC_SC_HALTED,
695 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
696/* 5-63 reserved*/
697};
698
699/*
700 * RXE Receive Error flags
701 */
702#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
703static struct flag_table rxe_err_status_flags[] = {
704/* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
705/* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
706/* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
707/* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
708/* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
709/* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
710/* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
711/* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
712/* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
713/* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
714/*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
715/*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
716/*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
717/*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
718/*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
719/*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
720/*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
721 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
722/*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
723/*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
724/*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
725 RXES(RBUF_BLOCK_LIST_READ_UNC)),
726/*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
727 RXES(RBUF_BLOCK_LIST_READ_COR)),
728/*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
729 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
730/*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
731 RXES(RBUF_CSR_QENT_CNT_PARITY)),
732/*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
733 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
734/*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
735 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
736/*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
737/*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
738/*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
739 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
740/*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
741/*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
742/*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
743/*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
744/*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
745/*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
746/*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
747/*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
748 RXES(RBUF_FL_INITDONE_PARITY)),
749/*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
750 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
751/*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
752/*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
753/*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
754/*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
755 RXES(LOOKUP_DES_PART1_UNC_COR)),
756/*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
757 RXES(LOOKUP_DES_PART2_PARITY)),
758/*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
759/*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
760/*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
761/*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
762/*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
763/*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
764/*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
765/*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
766/*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
767/*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
768/*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
769/*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
770/*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
771/*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
772/*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
773/*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
774/*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
775/*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
776/*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
777/*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
778/*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
779/*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
780};
781
782/* RXE errors that will trigger an SPC freeze */
783#define ALL_RXE_FREEZE_ERR \
784 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
785 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
786 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
787 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
788 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
789 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
790 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
791 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
792 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
793 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
794 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
795 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
823 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
824 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
825 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
826 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
827 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
828
829#define RXE_FREEZE_ABORT_MASK \
830 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
831 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
832 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
833
834/*
835 * DCC Error Flags
836 */
837#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
838static struct flag_table dcc_err_flags[] = {
839 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
840 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
841 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
842 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
843 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
844 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
845 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
846 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
847 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
848 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
849 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
850 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
851 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
852 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
853 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
854 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
855 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
856 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
857 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
858 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
859 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
860 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
861 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
862 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
863 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
864 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
865 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
866 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
867 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
868 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
869 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
870 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
871 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
872 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
873 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
874 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
875 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
876 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
877 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
878 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
879 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
880 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
881 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
882 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
883 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
884 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
885};
886
887/*
888 * LCB error flags
889 */
890#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
891static struct flag_table lcb_err_flags[] = {
892/* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
893/* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
894/* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
895/* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
896 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
897/* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
898/* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
899/* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
900/* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
901/* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
902/* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
903/*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
904/*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
905/*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
906/*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
907 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
908/*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
909/*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
910/*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
911/*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
912/*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
913/*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
914 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
915/*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
916/*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
917/*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
918/*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
919/*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
920/*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
921/*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
922 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
923/*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
924/*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
925 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
926/*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
927 LCBE(REDUNDANT_FLIT_PARITY_ERR))
928};
929
930/*
931 * DC8051 Error Flags
932 */
933#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
934static struct flag_table dc8051_err_flags[] = {
935 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
936 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
937 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
938 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
939 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
940 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
941 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
942 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
943 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
17fb4f29 944 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
77241056
MM		 945 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
946};
947
948/*
949 * DC8051 Information Error flags
950 *
951 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
952 */
953static struct flag_table dc8051_info_err_flags[] = {
954 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
955 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
956 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
957 FLAG_ENTRY0("Serdes internal loopback failure",
17fb4f29 958 FAILED_SERDES_INTERNAL_LOOPBACK),
77241056
MM		 959 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
960 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
961 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
962 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
963 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
964 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
965 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
966 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT)
967};
968
969/*
970 * DC8051 Information Host Information flags
971 *
972 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
973 */
974static struct flag_table dc8051_info_host_msg_flags[] = {
975 FLAG_ENTRY0("Host request done", 0x0001),
976 FLAG_ENTRY0("BC SMA message", 0x0002),
977 FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
978 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
979 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
980 FLAG_ENTRY0("External device config request", 0x0020),
981 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
982 FLAG_ENTRY0("LinkUp achieved", 0x0080),
983 FLAG_ENTRY0("Link going down", 0x0100),
984};
985
77241056
MM		 986static u32 encoded_size(u32 size);
987static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
988static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
989static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
990 u8 *continuous);
991static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
992 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
993static void read_vc_remote_link_width(struct hfi1_devdata *dd,
994 u8 *remote_tx_rate, u16 *link_widths);
995static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
996 u8 *flag_bits, u16 *link_widths);
997static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
998 u8 *device_rev);
999static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
1000static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1001static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1002 u8 *tx_polarity_inversion,
1003 u8 *rx_polarity_inversion, u8 *max_rate);
1004static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1005 unsigned int context, u64 err_status);
1006static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1007static void handle_dcc_err(struct hfi1_devdata *dd,
1008 unsigned int context, u64 err_status);
1009static void handle_lcb_err(struct hfi1_devdata *dd,
1010 unsigned int context, u64 err_status);
1011static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1012static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1013static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1014static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1015static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1016static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1017static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1018static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1019static void set_partition_keys(struct hfi1_pportdata *);
1020static const char *link_state_name(u32 state);
1021static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1022 u32 state);
1023static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1024 u64 *out_data);
1025static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1026static int thermal_init(struct hfi1_devdata *dd);
1027
1028static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1029 int msecs);
1030static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1031static void handle_temp_err(struct hfi1_devdata *);
1032static void dc_shutdown(struct hfi1_devdata *);
1033static void dc_start(struct hfi1_devdata *);
1034
1035/*
1036 * Error interrupt table entry. This is used as input to the interrupt
1037 * "clear down" routine used for all second tier error interrupt register.
1038 * Second tier interrupt registers have a single bit representing them
1039 * in the top-level CceIntStatus.
1040 */
1041struct err_reg_info {
1042 u32 status; /* status CSR offset */
1043 u32 clear; /* clear CSR offset */
1044 u32 mask; /* mask CSR offset */
1045 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1046 const char *desc;
1047};
1048
1049#define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1050#define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1051#define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1052
1053/*
1054 * Helpers for building HFI and DC error interrupt table entries. Different
1055 * helpers are needed because of inconsistent register names.
1056 */
1057#define EE(reg, handler, desc) \
1058 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1059 handler, desc }
1060#define DC_EE1(reg, handler, desc) \
1061 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1062#define DC_EE2(reg, handler, desc) \
1063 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1064
1065/*
1066 * Table of the "misc" grouping of error interrupts. Each entry refers to
1067 * another register containing more information.
1068 */
1069static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1070/* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1071/* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1072/* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1073/* 3*/ { 0, 0, 0, NULL }, /* reserved */
1074/* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1075/* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1076/* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1077/* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1078 /* the rest are reserved */
1079};
1080
1081/*
1082 * Index into the Various section of the interrupt sources
1083 * corresponding to the Critical Temperature interrupt.
1084 */
1085#define TCRIT_INT_SOURCE 4
1086
1087/*
1088 * SDMA error interrupt entry - refers to another register containing more
1089 * information.
1090 */
1091static const struct err_reg_info sdma_eng_err =
1092 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1093
1094static const struct err_reg_info various_err[NUM_VARIOUS] = {
1095/* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1096/* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1097/* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1098/* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1099/* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1100 /* rest are reserved */
1101};
1102
1103/*
1104 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1105 * register can not be derived from the MTU value because 10K is not
1106 * a power of 2. Therefore, we need a constant. Everything else can
1107 * be calculated.
1108 */
1109#define DCC_CFG_PORT_MTU_CAP_10240 7
1110
1111/*
1112 * Table of the DC grouping of error interrupts. Each entry refers to
1113 * another register containing more information.
1114 */
1115static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1116/* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1117/* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1118/* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1119/* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1120 /* the rest are reserved */
1121};
1122
1123struct cntr_entry {
1124 /*
1125 * counter name
1126 */
1127 char *name;
1128
1129 /*
1130 * csr to read for name (if applicable)
1131 */
1132 u64 csr;
1133
1134 /*
1135 * offset into dd or ppd to store the counter's value
1136 */
1137 int offset;
1138
1139 /*
1140 * flags
1141 */
1142 u8 flags;
1143
1144 /*
1145 * accessor for stat element, context either dd or ppd
1146 */
17fb4f29
JJ		 1147 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1148 int mode, u64 data);
77241056
MM		 1149};
1150
1151#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1152#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1153
1154#define CNTR_ELEM(name, csr, offset, flags, accessor) \
1155{ \
1156 name, \
1157 csr, \
1158 offset, \
1159 flags, \
1160 accessor \
1161}
1162
1163/* 32bit RXE */
1164#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1165CNTR_ELEM(#name, \
1166 (counter * 8 + RCV_COUNTER_ARRAY32), \
1167 0, flags | CNTR_32BIT, \
1168 port_access_u32_csr)
1169
1170#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1171CNTR_ELEM(#name, \
1172 (counter * 8 + RCV_COUNTER_ARRAY32), \
1173 0, flags | CNTR_32BIT, \
1174 dev_access_u32_csr)
1175
1176/* 64bit RXE */
1177#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1178CNTR_ELEM(#name, \
1179 (counter * 8 + RCV_COUNTER_ARRAY64), \
1180 0, flags, \
1181 port_access_u64_csr)
1182
1183#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1184CNTR_ELEM(#name, \
1185 (counter * 8 + RCV_COUNTER_ARRAY64), \
1186 0, flags, \
1187 dev_access_u64_csr)
1188
1189#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1190#define OVR_ELM(ctx) \
1191CNTR_ELEM("RcvHdrOvr" #ctx, \
8638b77f 1192 (RCV_HDR_OVFL_CNT + ctx * 0x100), \
77241056
MM		 1193 0, CNTR_NORMAL, port_access_u64_csr)
1194
1195/* 32bit TXE */
1196#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1197CNTR_ELEM(#name, \
1198 (counter * 8 + SEND_COUNTER_ARRAY32), \
1199 0, flags | CNTR_32BIT, \
1200 port_access_u32_csr)
1201
1202/* 64bit TXE */
1203#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1204CNTR_ELEM(#name, \
1205 (counter * 8 + SEND_COUNTER_ARRAY64), \
1206 0, flags, \
1207 port_access_u64_csr)
1208
1209# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1210CNTR_ELEM(#name,\
1211 counter * 8 + SEND_COUNTER_ARRAY64, \
1212 0, \
1213 flags, \
1214 dev_access_u64_csr)
1215
1216/* CCE */
1217#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1218CNTR_ELEM(#name, \
1219 (counter * 8 + CCE_COUNTER_ARRAY32), \
1220 0, flags | CNTR_32BIT, \
1221 dev_access_u32_csr)
1222
1223#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1224CNTR_ELEM(#name, \
1225 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1226 0, flags | CNTR_32BIT, \
1227 dev_access_u32_csr)
1228
1229/* DC */
1230#define DC_PERF_CNTR(name, counter, flags) \
1231CNTR_ELEM(#name, \
1232 counter, \
1233 0, \
1234 flags, \
1235 dev_access_u64_csr)
1236
1237#define DC_PERF_CNTR_LCB(name, counter, flags) \
1238CNTR_ELEM(#name, \
1239 counter, \
1240 0, \
1241 flags, \
1242 dc_access_lcb_cntr)
1243
1244/* ibp counters */
1245#define SW_IBP_CNTR(name, cntr) \
1246CNTR_ELEM(#name, \
1247 0, \
1248 0, \
1249 CNTR_SYNTH, \
1250 access_ibp_##cntr)
1251
1252u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1253{
1254 u64 val;
1255
1256 if (dd->flags & HFI1_PRESENT) {
1257 val = readq((void __iomem *)dd->kregbase + offset);
1258 return val;
1259 }
1260 return -1;
1261}
1262
1263void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1264{
1265 if (dd->flags & HFI1_PRESENT)
1266 writeq(value, (void __iomem *)dd->kregbase + offset);
1267}
1268
1269void __iomem *get_csr_addr(
1270 struct hfi1_devdata *dd,
1271 u32 offset)
1272{
1273 return (void __iomem *)dd->kregbase + offset;
1274}
1275
1276static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1277 int mode, u64 value)
1278{
1279 u64 ret;
1280
77241056
MM		 1281 if (mode == CNTR_MODE_R) {
1282 ret = read_csr(dd, csr);
1283 } else if (mode == CNTR_MODE_W) {
1284 write_csr(dd, csr, value);
1285 ret = value;
1286 } else {
1287 dd_dev_err(dd, "Invalid cntr register access mode");
1288 return 0;
1289 }
1290
1291 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1292 return ret;
1293}
1294
1295/* Dev Access */
1296static u64 dev_access_u32_csr(const struct cntr_entry *entry,
17fb4f29 1297 void *context, int vl, int mode, u64 data)
77241056 1298{
a787bde8 1299 struct hfi1_devdata *dd = context;
a699c6c2 1300 u64 csr = entry->csr;
77241056 1301
a699c6c2
VM		 1302 if (entry->flags & CNTR_SDMA) {
1303 if (vl == CNTR_INVALID_VL)
1304 return 0;
1305 csr += 0x100 * vl;
1306 } else {
1307 if (vl != CNTR_INVALID_VL)
1308 return 0;
1309 }
1310 return read_write_csr(dd, csr, mode, data);
1311}
1312
1313static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1314 void *context, int idx, int mode, u64 data)
1315{
1316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1317
1318 if (dd->per_sdma && idx < dd->num_sdma)
1319 return dd->per_sdma[idx].err_cnt;
1320 return 0;
1321}
1322
1323static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1324 void *context, int idx, int mode, u64 data)
1325{
1326 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1327
1328 if (dd->per_sdma && idx < dd->num_sdma)
1329 return dd->per_sdma[idx].sdma_int_cnt;
1330 return 0;
1331}
1332
1333static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1334 void *context, int idx, int mode, u64 data)
1335{
1336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1337
1338 if (dd->per_sdma && idx < dd->num_sdma)
1339 return dd->per_sdma[idx].idle_int_cnt;
1340 return 0;
1341}
1342
1343static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1344 void *context, int idx, int mode,
1345 u64 data)
1346{
1347 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1348
1349 if (dd->per_sdma && idx < dd->num_sdma)
1350 return dd->per_sdma[idx].progress_int_cnt;
1351 return 0;
77241056
MM		 1352}
1353
1354static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
17fb4f29 1355 int vl, int mode, u64 data)
77241056 1356{
a787bde8 1357 struct hfi1_devdata *dd = context;
77241056
MM		 1358
1359 u64 val = 0;
1360 u64 csr = entry->csr;
1361
1362 if (entry->flags & CNTR_VL) {
1363 if (vl == CNTR_INVALID_VL)
1364 return 0;
1365 csr += 8 * vl;
1366 } else {
1367 if (vl != CNTR_INVALID_VL)
1368 return 0;
1369 }
1370
1371 val = read_write_csr(dd, csr, mode, data);
1372 return val;
1373}
1374
1375static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
17fb4f29 1376 int vl, int mode, u64 data)
77241056 1377{
a787bde8 1378 struct hfi1_devdata *dd = context;
77241056
MM		 1379 u32 csr = entry->csr;
1380 int ret = 0;
1381
1382 if (vl != CNTR_INVALID_VL)
1383 return 0;
1384 if (mode == CNTR_MODE_R)
1385 ret = read_lcb_csr(dd, csr, &data);
1386 else if (mode == CNTR_MODE_W)
1387 ret = write_lcb_csr(dd, csr, data);
1388
1389 if (ret) {
1390 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1391 return 0;
1392 }
1393
1394 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1395 return data;
1396}
1397
1398/* Port Access */
1399static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
17fb4f29 1400 int vl, int mode, u64 data)
77241056 1401{
a787bde8 1402 struct hfi1_pportdata *ppd = context;
77241056
MM		 1403
1404 if (vl != CNTR_INVALID_VL)
1405 return 0;
1406 return read_write_csr(ppd->dd, entry->csr, mode, data);
1407}
1408
1409static u64 port_access_u64_csr(const struct cntr_entry *entry,
17fb4f29 1410 void *context, int vl, int mode, u64 data)
77241056 1411{
a787bde8 1412 struct hfi1_pportdata *ppd = context;
77241056
MM		 1413 u64 val;
1414 u64 csr = entry->csr;
1415
1416 if (entry->flags & CNTR_VL) {
1417 if (vl == CNTR_INVALID_VL)
1418 return 0;
1419 csr += 8 * vl;
1420 } else {
1421 if (vl != CNTR_INVALID_VL)
1422 return 0;
1423 }
1424 val = read_write_csr(ppd->dd, csr, mode, data);
1425 return val;
1426}
1427
1428/* Software defined */
1429static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1430 u64 data)
1431{
1432 u64 ret;
1433
1434 if (mode == CNTR_MODE_R) {
1435 ret = *cntr;
1436 } else if (mode == CNTR_MODE_W) {
1437 *cntr = data;
1438 ret = data;
1439 } else {
1440 dd_dev_err(dd, "Invalid cntr sw access mode");
1441 return 0;
1442 }
1443
1444 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1445
1446 return ret;
1447}
1448
1449static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
17fb4f29 1450 int vl, int mode, u64 data)
77241056 1451{
a787bde8 1452 struct hfi1_pportdata *ppd = context;
77241056
MM		 1453
1454 if (vl != CNTR_INVALID_VL)
1455 return 0;
1456 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1457}
1458
1459static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
17fb4f29 1460 int vl, int mode, u64 data)
77241056 1461{
a787bde8 1462 struct hfi1_pportdata *ppd = context;
77241056
MM		 1463
1464 if (vl != CNTR_INVALID_VL)
1465 return 0;
1466 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1467}
1468
6d014530
DL		 1469static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1470 void *context, int vl, int mode,
1471 u64 data)
1472{
1473 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1474
1475 if (vl != CNTR_INVALID_VL)
1476 return 0;
1477 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1478}
1479
77241056 1480static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
17fb4f29 1481 void *context, int vl, int mode, u64 data)
77241056 1482{
69a00b8e
MM		 1483 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1484 u64 zero = 0;
1485 u64 *counter;
77241056 1486
69a00b8e
MM		 1487 if (vl == CNTR_INVALID_VL)
1488 counter = &ppd->port_xmit_discards;
1489 else if (vl >= 0 && vl < C_VL_COUNT)
1490 counter = &ppd->port_xmit_discards_vl[vl];
1491 else
1492 counter = &zero;
77241056 1493
69a00b8e 1494 return read_write_sw(ppd->dd, counter, mode, data);
77241056
MM		 1495}
1496
1497static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
17fb4f29
JJ		 1498 void *context, int vl, int mode,
1499 u64 data)
77241056 1500{
a787bde8 1501 struct hfi1_pportdata *ppd = context;
77241056
MM		 1502
1503 if (vl != CNTR_INVALID_VL)
1504 return 0;
1505
1506 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1507 mode, data);
1508}
1509
1510static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
17fb4f29 1511 void *context, int vl, int mode, u64 data)
77241056 1512{
a787bde8 1513 struct hfi1_pportdata *ppd = context;
77241056
MM		 1514
1515 if (vl != CNTR_INVALID_VL)
1516 return 0;
1517
1518 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1519 mode, data);
1520}
1521
1522u64 get_all_cpu_total(u64 __percpu *cntr)
1523{
1524 int cpu;
1525 u64 counter = 0;
1526
1527 for_each_possible_cpu(cpu)
1528 counter += *per_cpu_ptr(cntr, cpu);
1529 return counter;
1530}
1531
1532static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1533 u64 __percpu *cntr,
1534 int vl, int mode, u64 data)
1535{
77241056
MM		 1536 u64 ret = 0;
1537
1538 if (vl != CNTR_INVALID_VL)
1539 return 0;
1540
1541 if (mode == CNTR_MODE_R) {
1542 ret = get_all_cpu_total(cntr) - *z_val;
1543 } else if (mode == CNTR_MODE_W) {
1544 /* A write can only zero the counter */
1545 if (data == 0)
1546 *z_val = get_all_cpu_total(cntr);
1547 else
1548 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1549 } else {
1550 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1551 return 0;
1552 }
1553
1554 return ret;
1555}
1556
1557static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1558 void *context, int vl, int mode, u64 data)
1559{
a787bde8 1560 struct hfi1_devdata *dd = context;
77241056
MM		 1561
1562 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1563 mode, data);
1564}
1565
1566static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
17fb4f29 1567 void *context, int vl, int mode, u64 data)
77241056 1568{
a787bde8 1569 struct hfi1_devdata *dd = context;
77241056
MM		 1570
1571 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1572 mode, data);
1573}
1574
1575static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1576 void *context, int vl, int mode, u64 data)
1577{
a787bde8 1578 struct hfi1_devdata *dd = context;
77241056
MM		 1579
1580 return dd->verbs_dev.n_piowait;
1581}
1582
14553ca1
MM		 1583static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1584 void *context, int vl, int mode, u64 data)
1585{
1586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1587
1588 return dd->verbs_dev.n_piodrain;
1589}
1590
77241056
MM		 1591static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1592 void *context, int vl, int mode, u64 data)
1593{
a787bde8 1594 struct hfi1_devdata *dd = context;
77241056
MM		 1595
1596 return dd->verbs_dev.n_txwait;
1597}
1598
1599static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1600 void *context, int vl, int mode, u64 data)
1601{
a787bde8 1602 struct hfi1_devdata *dd = context;
77241056
MM		 1603
1604 return dd->verbs_dev.n_kmem_wait;
1605}
1606
b421922e 1607static u64 access_sw_send_schedule(const struct cntr_entry *entry,
17fb4f29 1608 void *context, int vl, int mode, u64 data)
b421922e
DL		 1609{
1610 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1611
89abfc8d
VM		 1612 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1613 mode, data);
b421922e
DL		 1614}
1615
2c5b521a
JR		 1616/* Software counters for the error status bits within MISC_ERR_STATUS */
1617static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1618 void *context, int vl, int mode,
1619 u64 data)
1620{
1621 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1622
1623 return dd->misc_err_status_cnt[12];
1624}
1625
1626static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1627 void *context, int vl, int mode,
1628 u64 data)
1629{
1630 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1631
1632 return dd->misc_err_status_cnt[11];
1633}
1634
1635static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1636 void *context, int vl, int mode,
1637 u64 data)
1638{
1639 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1640
1641 return dd->misc_err_status_cnt[10];
1642}
1643
1644static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1645 void *context, int vl,
1646 int mode, u64 data)
1647{
1648 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1649
1650 return dd->misc_err_status_cnt[9];
1651}
1652
1653static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1654 void *context, int vl, int mode,
1655 u64 data)
1656{
1657 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1658
1659 return dd->misc_err_status_cnt[8];
1660}
1661
1662static u64 access_misc_efuse_read_bad_addr_err_cnt(
1663 const struct cntr_entry *entry,
1664 void *context, int vl, int mode, u64 data)
1665{
1666 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1667
1668 return dd->misc_err_status_cnt[7];
1669}
1670
1671static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1672 void *context, int vl,
1673 int mode, u64 data)
1674{
1675 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1676
1677 return dd->misc_err_status_cnt[6];
1678}
1679
1680static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1681 void *context, int vl, int mode,
1682 u64 data)
1683{
1684 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1685
1686 return dd->misc_err_status_cnt[5];
1687}
1688
1689static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1690 void *context, int vl, int mode,
1691 u64 data)
1692{
1693 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1694
1695 return dd->misc_err_status_cnt[4];
1696}
1697
1698static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1699 void *context, int vl,
1700 int mode, u64 data)
1701{
1702 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1703
1704 return dd->misc_err_status_cnt[3];
1705}
1706
1707static u64 access_misc_csr_write_bad_addr_err_cnt(
1708 const struct cntr_entry *entry,
1709 void *context, int vl, int mode, u64 data)
1710{
1711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1712
1713 return dd->misc_err_status_cnt[2];
1714}
1715
1716static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1717 void *context, int vl,
1718 int mode, u64 data)
1719{
1720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1721
1722 return dd->misc_err_status_cnt[1];
1723}
1724
1725static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1726 void *context, int vl, int mode,
1727 u64 data)
1728{
1729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1730
1731 return dd->misc_err_status_cnt[0];
1732}
1733
1734/*
1735 * Software counter for the aggregate of
1736 * individual CceErrStatus counters
1737 */
1738static u64 access_sw_cce_err_status_aggregated_cnt(
1739 const struct cntr_entry *entry,
1740 void *context, int vl, int mode, u64 data)
1741{
1742 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1743
1744 return dd->sw_cce_err_status_aggregate;
1745}
1746
1747/*
1748 * Software counters corresponding to each of the
1749 * error status bits within CceErrStatus
1750 */
1751static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1752 void *context, int vl, int mode,
1753 u64 data)
1754{
1755 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1756
1757 return dd->cce_err_status_cnt[40];
1758}
1759
1760static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1761 void *context, int vl, int mode,
1762 u64 data)
1763{
1764 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1765
1766 return dd->cce_err_status_cnt[39];
1767}
1768
1769static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1770 void *context, int vl, int mode,
1771 u64 data)
1772{
1773 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1774
1775 return dd->cce_err_status_cnt[38];
1776}
1777
1778static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1779 void *context, int vl, int mode,
1780 u64 data)
1781{
1782 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1783
1784 return dd->cce_err_status_cnt[37];
1785}
1786
1787static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1788 void *context, int vl, int mode,
1789 u64 data)
1790{
1791 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1792
1793 return dd->cce_err_status_cnt[36];
1794}
1795
1796static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1797 const struct cntr_entry *entry,
1798 void *context, int vl, int mode, u64 data)
1799{
1800 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1801
1802 return dd->cce_err_status_cnt[35];
1803}
1804
1805static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1806 const struct cntr_entry *entry,
1807 void *context, int vl, int mode, u64 data)
1808{
1809 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1810
1811 return dd->cce_err_status_cnt[34];
1812}
1813
1814static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1815 void *context, int vl,
1816 int mode, u64 data)
1817{
1818 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1819
1820 return dd->cce_err_status_cnt[33];
1821}
1822
1823static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1824 void *context, int vl, int mode,
1825 u64 data)
1826{
1827 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1828
1829 return dd->cce_err_status_cnt[32];
1830}
1831
1832static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1833 void *context, int vl, int mode, u64 data)
1834{
1835 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1836
1837 return dd->cce_err_status_cnt[31];
1838}
1839
1840static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1841 void *context, int vl, int mode,
1842 u64 data)
1843{
1844 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1845
1846 return dd->cce_err_status_cnt[30];
1847}
1848
1849static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1850 void *context, int vl, int mode,
1851 u64 data)
1852{
1853 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1854
1855 return dd->cce_err_status_cnt[29];
1856}
1857
1858static u64 access_pcic_transmit_back_parity_err_cnt(
1859 const struct cntr_entry *entry,
1860 void *context, int vl, int mode, u64 data)
1861{
1862 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1863
1864 return dd->cce_err_status_cnt[28];
1865}
1866
1867static u64 access_pcic_transmit_front_parity_err_cnt(
1868 const struct cntr_entry *entry,
1869 void *context, int vl, int mode, u64 data)
1870{
1871 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1872
1873 return dd->cce_err_status_cnt[27];
1874}
1875
1876static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1877 void *context, int vl, int mode,
1878 u64 data)
1879{
1880 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1881
1882 return dd->cce_err_status_cnt[26];
1883}
1884
1885static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1886 void *context, int vl, int mode,
1887 u64 data)
1888{
1889 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1890
1891 return dd->cce_err_status_cnt[25];
1892}
1893
1894static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1895 void *context, int vl, int mode,
1896 u64 data)
1897{
1898 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1899
1900 return dd->cce_err_status_cnt[24];
1901}
1902
1903static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1904 void *context, int vl, int mode,
1905 u64 data)
1906{
1907 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1908
1909 return dd->cce_err_status_cnt[23];
1910}
1911
1912static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1913 void *context, int vl,
1914 int mode, u64 data)
1915{
1916 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1917
1918 return dd->cce_err_status_cnt[22];
1919}
1920
1921static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
1922 void *context, int vl, int mode,
1923 u64 data)
1924{
1925 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1926
1927 return dd->cce_err_status_cnt[21];
1928}
1929
1930static u64 access_pcic_n_post_dat_q_parity_err_cnt(
1931 const struct cntr_entry *entry,
1932 void *context, int vl, int mode, u64 data)
1933{
1934 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1935
1936 return dd->cce_err_status_cnt[20];
1937}
1938
1939static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
1940 void *context, int vl,
1941 int mode, u64 data)
1942{
1943 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1944
1945 return dd->cce_err_status_cnt[19];
1946}
1947
1948static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1949 void *context, int vl, int mode,
1950 u64 data)
1951{
1952 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1953
1954 return dd->cce_err_status_cnt[18];
1955}
1956
1957static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1958 void *context, int vl, int mode,
1959 u64 data)
1960{
1961 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1962
1963 return dd->cce_err_status_cnt[17];
1964}
1965
1966static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1967 void *context, int vl, int mode,
1968 u64 data)
1969{
1970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1971
1972 return dd->cce_err_status_cnt[16];
1973}
1974
1975static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1976 void *context, int vl, int mode,
1977 u64 data)
1978{
1979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1980
1981 return dd->cce_err_status_cnt[15];
1982}
1983
1984static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
1985 void *context, int vl,
1986 int mode, u64 data)
1987{
1988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1989
1990 return dd->cce_err_status_cnt[14];
1991}
1992
1993static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
1994 void *context, int vl, int mode,
1995 u64 data)
1996{
1997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1998
1999 return dd->cce_err_status_cnt[13];
2000}
2001
2002static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2003 const struct cntr_entry *entry,
2004 void *context, int vl, int mode, u64 data)
2005{
2006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2007
2008 return dd->cce_err_status_cnt[12];
2009}
2010
2011static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2012 const struct cntr_entry *entry,
2013 void *context, int vl, int mode, u64 data)
2014{
2015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2016
2017 return dd->cce_err_status_cnt[11];
2018}
2019
2020static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2021 const struct cntr_entry *entry,
2022 void *context, int vl, int mode, u64 data)
2023{
2024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2025
2026 return dd->cce_err_status_cnt[10];
2027}
2028
2029static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2030 const struct cntr_entry *entry,
2031 void *context, int vl, int mode, u64 data)
2032{
2033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2034
2035 return dd->cce_err_status_cnt[9];
2036}
2037
2038static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2039 const struct cntr_entry *entry,
2040 void *context, int vl, int mode, u64 data)
2041{
2042 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2043
2044 return dd->cce_err_status_cnt[8];
2045}
2046
2047static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2048 void *context, int vl,
2049 int mode, u64 data)
2050{
2051 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2052
2053 return dd->cce_err_status_cnt[7];
2054}
2055
2056static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2057 const struct cntr_entry *entry,
2058 void *context, int vl, int mode, u64 data)
2059{
2060 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2061
2062 return dd->cce_err_status_cnt[6];
2063}
2064
2065static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2066 void *context, int vl, int mode,
2067 u64 data)
2068{
2069 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2070
2071 return dd->cce_err_status_cnt[5];
2072}
2073
2074static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2075 void *context, int vl, int mode,
2076 u64 data)
2077{
2078 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2079
2080 return dd->cce_err_status_cnt[4];
2081}
2082
2083static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2084 const struct cntr_entry *entry,
2085 void *context, int vl, int mode, u64 data)
2086{
2087 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2088
2089 return dd->cce_err_status_cnt[3];
2090}
2091
2092static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2093 void *context, int vl,
2094 int mode, u64 data)
2095{
2096 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2097
2098 return dd->cce_err_status_cnt[2];
2099}
2100
2101static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2102 void *context, int vl,
2103 int mode, u64 data)
2104{
2105 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2106
2107 return dd->cce_err_status_cnt[1];
2108}
2109
2110static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2111 void *context, int vl, int mode,
2112 u64 data)
2113{
2114 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2115
2116 return dd->cce_err_status_cnt[0];
2117}
2118
2119/*
2120 * Software counters corresponding to each of the
2121 * error status bits within RcvErrStatus
2122 */
2123static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2124 void *context, int vl, int mode,
2125 u64 data)
2126{
2127 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2128
2129 return dd->rcv_err_status_cnt[63];
2130}
2131
2132static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2133 void *context, int vl,
2134 int mode, u64 data)
2135{
2136 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2137
2138 return dd->rcv_err_status_cnt[62];
2139}
2140
2141static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2142 void *context, int vl, int mode,
2143 u64 data)
2144{
2145 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2146
2147 return dd->rcv_err_status_cnt[61];
2148}
2149
2150static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2151 void *context, int vl, int mode,
2152 u64 data)
2153{
2154 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2155
2156 return dd->rcv_err_status_cnt[60];
2157}
2158
2159static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2160 void *context, int vl,
2161 int mode, u64 data)
2162{
2163 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2164
2165 return dd->rcv_err_status_cnt[59];
2166}
2167
2168static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2169 void *context, int vl,
2170 int mode, u64 data)
2171{
2172 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2173
2174 return dd->rcv_err_status_cnt[58];
2175}
2176
2177static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2178 void *context, int vl, int mode,
2179 u64 data)
2180{
2181 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2182
2183 return dd->rcv_err_status_cnt[57];
2184}
2185
2186static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2187 void *context, int vl, int mode,
2188 u64 data)
2189{
2190 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2191
2192 return dd->rcv_err_status_cnt[56];
2193}
2194
2195static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2196 void *context, int vl, int mode,
2197 u64 data)
2198{
2199 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2200
2201 return dd->rcv_err_status_cnt[55];
2202}
2203
2204static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2205 const struct cntr_entry *entry,
2206 void *context, int vl, int mode, u64 data)
2207{
2208 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2209
2210 return dd->rcv_err_status_cnt[54];
2211}
2212
2213static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2214 const struct cntr_entry *entry,
2215 void *context, int vl, int mode, u64 data)
2216{
2217 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2218
2219 return dd->rcv_err_status_cnt[53];
2220}
2221
2222static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2223 void *context, int vl,
2224 int mode, u64 data)
2225{
2226 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2227
2228 return dd->rcv_err_status_cnt[52];
2229}
2230
2231static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2232 void *context, int vl,
2233 int mode, u64 data)
2234{
2235 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2236
2237 return dd->rcv_err_status_cnt[51];
2238}
2239
2240static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2241 void *context, int vl,
2242 int mode, u64 data)
2243{
2244 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2245
2246 return dd->rcv_err_status_cnt[50];
2247}
2248
2249static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2250 void *context, int vl,
2251 int mode, u64 data)
2252{
2253 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2254
2255 return dd->rcv_err_status_cnt[49];
2256}
2257
2258static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2259 void *context, int vl,
2260 int mode, u64 data)
2261{
2262 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2263
2264 return dd->rcv_err_status_cnt[48];
2265}
2266
2267static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2268 void *context, int vl,
2269 int mode, u64 data)
2270{
2271 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2272
2273 return dd->rcv_err_status_cnt[47];
2274}
2275
2276static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2277 void *context, int vl, int mode,
2278 u64 data)
2279{
2280 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2281
2282 return dd->rcv_err_status_cnt[46];
2283}
2284
2285static u64 access_rx_hq_intr_csr_parity_err_cnt(
2286 const struct cntr_entry *entry,
2287 void *context, int vl, int mode, u64 data)
2288{
2289 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2290
2291 return dd->rcv_err_status_cnt[45];
2292}
2293
2294static u64 access_rx_lookup_csr_parity_err_cnt(
2295 const struct cntr_entry *entry,
2296 void *context, int vl, int mode, u64 data)
2297{
2298 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2299
2300 return dd->rcv_err_status_cnt[44];
2301}
2302
2303static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2304 const struct cntr_entry *entry,
2305 void *context, int vl, int mode, u64 data)
2306{
2307 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2308
2309 return dd->rcv_err_status_cnt[43];
2310}
2311
2312static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2313 const struct cntr_entry *entry,
2314 void *context, int vl, int mode, u64 data)
2315{
2316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2317
2318 return dd->rcv_err_status_cnt[42];
2319}
2320
2321static u64 access_rx_lookup_des_part2_parity_err_cnt(
2322 const struct cntr_entry *entry,
2323 void *context, int vl, int mode, u64 data)
2324{
2325 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2326
2327 return dd->rcv_err_status_cnt[41];
2328}
2329
2330static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2331 const struct cntr_entry *entry,
2332 void *context, int vl, int mode, u64 data)
2333{
2334 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2335
2336 return dd->rcv_err_status_cnt[40];
2337}
2338
2339static u64 access_rx_lookup_des_part1_unc_err_cnt(
2340 const struct cntr_entry *entry,
2341 void *context, int vl, int mode, u64 data)
2342{
2343 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2344
2345 return dd->rcv_err_status_cnt[39];
2346}
2347
2348static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2349 const struct cntr_entry *entry,
2350 void *context, int vl, int mode, u64 data)
2351{
2352 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2353
2354 return dd->rcv_err_status_cnt[38];
2355}
2356
2357static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2358 const struct cntr_entry *entry,
2359 void *context, int vl, int mode, u64 data)
2360{
2361 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2362
2363 return dd->rcv_err_status_cnt[37];
2364}
2365
2366static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2367 const struct cntr_entry *entry,
2368 void *context, int vl, int mode, u64 data)
2369{
2370 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2371
2372 return dd->rcv_err_status_cnt[36];
2373}
2374
2375static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2376 const struct cntr_entry *entry,
2377 void *context, int vl, int mode, u64 data)
2378{
2379 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2380
2381 return dd->rcv_err_status_cnt[35];
2382}
2383
2384static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2385 const struct cntr_entry *entry,
2386 void *context, int vl, int mode, u64 data)
2387{
2388 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2389
2390 return dd->rcv_err_status_cnt[34];
2391}
2392
2393static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2394 const struct cntr_entry *entry,
2395 void *context, int vl, int mode, u64 data)
2396{
2397 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2398
2399 return dd->rcv_err_status_cnt[33];
2400}
2401
2402static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2403 void *context, int vl, int mode,
2404 u64 data)
2405{
2406 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2407
2408 return dd->rcv_err_status_cnt[32];
2409}
2410
2411static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2412 void *context, int vl, int mode,
2413 u64 data)
2414{
2415 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2416
2417 return dd->rcv_err_status_cnt[31];
2418}
2419
2420static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2421 void *context, int vl, int mode,
2422 u64 data)
2423{
2424 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2425
2426 return dd->rcv_err_status_cnt[30];
2427}
2428
2429static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2430 void *context, int vl, int mode,
2431 u64 data)
2432{
2433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2434
2435 return dd->rcv_err_status_cnt[29];
2436}
2437
2438static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2439 void *context, int vl,
2440 int mode, u64 data)
2441{
2442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2443
2444 return dd->rcv_err_status_cnt[28];
2445}
2446
2447static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2448 const struct cntr_entry *entry,
2449 void *context, int vl, int mode, u64 data)
2450{
2451 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2452
2453 return dd->rcv_err_status_cnt[27];
2454}
2455
2456static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2457 const struct cntr_entry *entry,
2458 void *context, int vl, int mode, u64 data)
2459{
2460 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2461
2462 return dd->rcv_err_status_cnt[26];
2463}
2464
2465static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2466 const struct cntr_entry *entry,
2467 void *context, int vl, int mode, u64 data)
2468{
2469 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2470
2471 return dd->rcv_err_status_cnt[25];
2472}
2473
2474static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2475 const struct cntr_entry *entry,
2476 void *context, int vl, int mode, u64 data)
2477{
2478 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2479
2480 return dd->rcv_err_status_cnt[24];
2481}
2482
2483static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2484 const struct cntr_entry *entry,
2485 void *context, int vl, int mode, u64 data)
2486{
2487 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2488
2489 return dd->rcv_err_status_cnt[23];
2490}
2491
2492static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2493 const struct cntr_entry *entry,
2494 void *context, int vl, int mode, u64 data)
2495{
2496 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2497
2498 return dd->rcv_err_status_cnt[22];
2499}
2500
2501static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2502 const struct cntr_entry *entry,
2503 void *context, int vl, int mode, u64 data)
2504{
2505 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2506
2507 return dd->rcv_err_status_cnt[21];
2508}
2509
2510static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2511 const struct cntr_entry *entry,
2512 void *context, int vl, int mode, u64 data)
2513{
2514 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2515
2516 return dd->rcv_err_status_cnt[20];
2517}
2518
2519static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2520 const struct cntr_entry *entry,
2521 void *context, int vl, int mode, u64 data)
2522{
2523 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2524
2525 return dd->rcv_err_status_cnt[19];
2526}
2527
2528static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2529 void *context, int vl,
2530 int mode, u64 data)
2531{
2532 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2533
2534 return dd->rcv_err_status_cnt[18];
2535}
2536
2537static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2538 void *context, int vl,
2539 int mode, u64 data)
2540{
2541 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2542
2543 return dd->rcv_err_status_cnt[17];
2544}
2545
2546static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2547 const struct cntr_entry *entry,
2548 void *context, int vl, int mode, u64 data)
2549{
2550 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2551
2552 return dd->rcv_err_status_cnt[16];
2553}
2554
2555static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2556 const struct cntr_entry *entry,
2557 void *context, int vl, int mode, u64 data)
2558{
2559 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2560
2561 return dd->rcv_err_status_cnt[15];
2562}
2563
2564static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2565 void *context, int vl,
2566 int mode, u64 data)
2567{
2568 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2569
2570 return dd->rcv_err_status_cnt[14];
2571}
2572
2573static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2574 void *context, int vl,
2575 int mode, u64 data)
2576{
2577 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2578
2579 return dd->rcv_err_status_cnt[13];
2580}
2581
2582static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2583 void *context, int vl, int mode,
2584 u64 data)
2585{
2586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2587
2588 return dd->rcv_err_status_cnt[12];
2589}
2590
2591static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2592 void *context, int vl, int mode,
2593 u64 data)
2594{
2595 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2596
2597 return dd->rcv_err_status_cnt[11];
2598}
2599
2600static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2601 void *context, int vl, int mode,
2602 u64 data)
2603{
2604 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2605
2606 return dd->rcv_err_status_cnt[10];
2607}
2608
2609static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2610 void *context, int vl, int mode,
2611 u64 data)
2612{
2613 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2614
2615 return dd->rcv_err_status_cnt[9];
2616}
2617
2618static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2619 void *context, int vl, int mode,
2620 u64 data)
2621{
2622 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2623
2624 return dd->rcv_err_status_cnt[8];
2625}
2626
2627static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2628 const struct cntr_entry *entry,
2629 void *context, int vl, int mode, u64 data)
2630{
2631 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2632
2633 return dd->rcv_err_status_cnt[7];
2634}
2635
2636static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2637 const struct cntr_entry *entry,
2638 void *context, int vl, int mode, u64 data)
2639{
2640 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2641
2642 return dd->rcv_err_status_cnt[6];
2643}
2644
2645static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2646 void *context, int vl, int mode,
2647 u64 data)
2648{
2649 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2650
2651 return dd->rcv_err_status_cnt[5];
2652}
2653
2654static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2655 void *context, int vl, int mode,
2656 u64 data)
2657{
2658 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2659
2660 return dd->rcv_err_status_cnt[4];
2661}
2662
2663static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2664 void *context, int vl, int mode,
2665 u64 data)
2666{
2667 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2668
2669 return dd->rcv_err_status_cnt[3];
2670}
2671
2672static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2673 void *context, int vl, int mode,
2674 u64 data)
2675{
2676 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2677
2678 return dd->rcv_err_status_cnt[2];
2679}
2680
2681static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2682 void *context, int vl, int mode,
2683 u64 data)
2684{
2685 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2686
2687 return dd->rcv_err_status_cnt[1];
2688}
2689
2690static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2691 void *context, int vl, int mode,
2692 u64 data)
2693{
2694 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2695
2696 return dd->rcv_err_status_cnt[0];
2697}
2698
2699/*
2700 * Software counters corresponding to each of the
2701 * error status bits within SendPioErrStatus
2702 */
2703static u64 access_pio_pec_sop_head_parity_err_cnt(
2704 const struct cntr_entry *entry,
2705 void *context, int vl, int mode, u64 data)
2706{
2707 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2708
2709 return dd->send_pio_err_status_cnt[35];
2710}
2711
2712static u64 access_pio_pcc_sop_head_parity_err_cnt(
2713 const struct cntr_entry *entry,
2714 void *context, int vl, int mode, u64 data)
2715{
2716 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2717
2718 return dd->send_pio_err_status_cnt[34];
2719}
2720
2721static u64 access_pio_last_returned_cnt_parity_err_cnt(
2722 const struct cntr_entry *entry,
2723 void *context, int vl, int mode, u64 data)
2724{
2725 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2726
2727 return dd->send_pio_err_status_cnt[33];
2728}
2729
2730static u64 access_pio_current_free_cnt_parity_err_cnt(
2731 const struct cntr_entry *entry,
2732 void *context, int vl, int mode, u64 data)
2733{
2734 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2735
2736 return dd->send_pio_err_status_cnt[32];
2737}
2738
2739static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2740 void *context, int vl, int mode,
2741 u64 data)
2742{
2743 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2744
2745 return dd->send_pio_err_status_cnt[31];
2746}
2747
2748static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2749 void *context, int vl, int mode,
2750 u64 data)
2751{
2752 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2753
2754 return dd->send_pio_err_status_cnt[30];
2755}
2756
2757static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2758 void *context, int vl, int mode,
2759 u64 data)
2760{
2761 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2762
2763 return dd->send_pio_err_status_cnt[29];
2764}
2765
2766static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2767 const struct cntr_entry *entry,
2768 void *context, int vl, int mode, u64 data)
2769{
2770 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2771
2772 return dd->send_pio_err_status_cnt[28];
2773}
2774
2775static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2776 void *context, int vl, int mode,
2777 u64 data)
2778{
2779 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2780
2781 return dd->send_pio_err_status_cnt[27];
2782}
2783
2784static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2785 void *context, int vl, int mode,
2786 u64 data)
2787{
2788 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2789
2790 return dd->send_pio_err_status_cnt[26];
2791}
2792
2793static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2794 void *context, int vl,
2795 int mode, u64 data)
2796{
2797 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2798
2799 return dd->send_pio_err_status_cnt[25];
2800}
2801
2802static u64 access_pio_block_qw_count_parity_err_cnt(
2803 const struct cntr_entry *entry,
2804 void *context, int vl, int mode, u64 data)
2805{
2806 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2807
2808 return dd->send_pio_err_status_cnt[24];
2809}
2810
2811static u64 access_pio_write_qw_valid_parity_err_cnt(
2812 const struct cntr_entry *entry,
2813 void *context, int vl, int mode, u64 data)
2814{
2815 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2816
2817 return dd->send_pio_err_status_cnt[23];
2818}
2819
2820static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2821 void *context, int vl, int mode,
2822 u64 data)
2823{
2824 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2825
2826 return dd->send_pio_err_status_cnt[22];
2827}
2828
2829static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2830 void *context, int vl,
2831 int mode, u64 data)
2832{
2833 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2834
2835 return dd->send_pio_err_status_cnt[21];
2836}
2837
2838static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2839 void *context, int vl,
2840 int mode, u64 data)
2841{
2842 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2843
2844 return dd->send_pio_err_status_cnt[20];
2845}
2846
2847static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2848 void *context, int vl,
2849 int mode, u64 data)
2850{
2851 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2852
2853 return dd->send_pio_err_status_cnt[19];
2854}
2855
2856static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2857 const struct cntr_entry *entry,
2858 void *context, int vl, int mode, u64 data)
2859{
2860 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2861
2862 return dd->send_pio_err_status_cnt[18];
2863}
2864
2865static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2866 void *context, int vl, int mode,
2867 u64 data)
2868{
2869 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2870
2871 return dd->send_pio_err_status_cnt[17];
2872}
2873
2874static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2875 void *context, int vl, int mode,
2876 u64 data)
2877{
2878 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2879
2880 return dd->send_pio_err_status_cnt[16];
2881}
2882
2883static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2884 const struct cntr_entry *entry,
2885 void *context, int vl, int mode, u64 data)
2886{
2887 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2888
2889 return dd->send_pio_err_status_cnt[15];
2890}
2891
2892static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2893 const struct cntr_entry *entry,
2894 void *context, int vl, int mode, u64 data)
2895{
2896 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2897
2898 return dd->send_pio_err_status_cnt[14];
2899}
2900
2901static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2902 const struct cntr_entry *entry,
2903 void *context, int vl, int mode, u64 data)
2904{
2905 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2906
2907 return dd->send_pio_err_status_cnt[13];
2908}
2909
2910static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2911 const struct cntr_entry *entry,
2912 void *context, int vl, int mode, u64 data)
2913{
2914 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2915
2916 return dd->send_pio_err_status_cnt[12];
2917}
2918
2919static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
2920 const struct cntr_entry *entry,
2921 void *context, int vl, int mode, u64 data)
2922{
2923 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2924
2925 return dd->send_pio_err_status_cnt[11];
2926}
2927
2928static u64 access_pio_sm_pkt_reset_parity_err_cnt(
2929 const struct cntr_entry *entry,
2930 void *context, int vl, int mode, u64 data)
2931{
2932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2933
2934 return dd->send_pio_err_status_cnt[10];
2935}
2936
2937static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
2938 const struct cntr_entry *entry,
2939 void *context, int vl, int mode, u64 data)
2940{
2941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2942
2943 return dd->send_pio_err_status_cnt[9];
2944}
2945
2946static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
2947 const struct cntr_entry *entry,
2948 void *context, int vl, int mode, u64 data)
2949{
2950 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2951
2952 return dd->send_pio_err_status_cnt[8];
2953}
2954
2955static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
2956 const struct cntr_entry *entry,
2957 void *context, int vl, int mode, u64 data)
2958{
2959 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2960
2961 return dd->send_pio_err_status_cnt[7];
2962}
2963
2964static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
2965 void *context, int vl, int mode,
2966 u64 data)
2967{
2968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2969
2970 return dd->send_pio_err_status_cnt[6];
2971}
2972
2973static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
2974 void *context, int vl, int mode,
2975 u64 data)
2976{
2977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2978
2979 return dd->send_pio_err_status_cnt[5];
2980}
2981
2982static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
2983 void *context, int vl, int mode,
2984 u64 data)
2985{
2986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2987
2988 return dd->send_pio_err_status_cnt[4];
2989}
2990
2991static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
2992 void *context, int vl, int mode,
2993 u64 data)
2994{
2995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2996
2997 return dd->send_pio_err_status_cnt[3];
2998}
2999
3000static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3001 void *context, int vl, int mode,
3002 u64 data)
3003{
3004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3005
3006 return dd->send_pio_err_status_cnt[2];
3007}
3008
3009static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3010 void *context, int vl,
3011 int mode, u64 data)
3012{
3013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3014
3015 return dd->send_pio_err_status_cnt[1];
3016}
3017
3018static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3019 void *context, int vl, int mode,
3020 u64 data)
3021{
3022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3023
3024 return dd->send_pio_err_status_cnt[0];
3025}
3026
3027/*
3028 * Software counters corresponding to each of the
3029 * error status bits within SendDmaErrStatus
3030 */
3031static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3032 const struct cntr_entry *entry,
3033 void *context, int vl, int mode, u64 data)
3034{
3035 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3036
3037 return dd->send_dma_err_status_cnt[3];
3038}
3039
3040static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3041 const struct cntr_entry *entry,
3042 void *context, int vl, int mode, u64 data)
3043{
3044 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3045
3046 return dd->send_dma_err_status_cnt[2];
3047}
3048
3049static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3050 void *context, int vl, int mode,
3051 u64 data)
3052{
3053 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3054
3055 return dd->send_dma_err_status_cnt[1];
3056}
3057
3058static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3059 void *context, int vl, int mode,
3060 u64 data)
3061{
3062 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3063
3064 return dd->send_dma_err_status_cnt[0];
3065}
3066
3067/*
3068 * Software counters corresponding to each of the
3069 * error status bits within SendEgressErrStatus
3070 */
3071static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3072 const struct cntr_entry *entry,
3073 void *context, int vl, int mode, u64 data)
3074{
3075 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3076
3077 return dd->send_egress_err_status_cnt[63];
3078}
3079
3080static u64 access_tx_read_sdma_memory_csr_err_cnt(
3081 const struct cntr_entry *entry,
3082 void *context, int vl, int mode, u64 data)
3083{
3084 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3085
3086 return dd->send_egress_err_status_cnt[62];
3087}
3088
3089static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3090 void *context, int vl, int mode,
3091 u64 data)
3092{
3093 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3094
3095 return dd->send_egress_err_status_cnt[61];
3096}
3097
3098static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3099 void *context, int vl,
3100 int mode, u64 data)
3101{
3102 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3103
3104 return dd->send_egress_err_status_cnt[60];
3105}
3106
3107static u64 access_tx_read_sdma_memory_cor_err_cnt(
3108 const struct cntr_entry *entry,
3109 void *context, int vl, int mode, u64 data)
3110{
3111 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3112
3113 return dd->send_egress_err_status_cnt[59];
3114}
3115
3116static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3117 void *context, int vl, int mode,
3118 u64 data)
3119{
3120 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3121
3122 return dd->send_egress_err_status_cnt[58];
3123}
3124
3125static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3126 void *context, int vl, int mode,
3127 u64 data)
3128{
3129 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3130
3131 return dd->send_egress_err_status_cnt[57];
3132}
3133
3134static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3135 void *context, int vl, int mode,
3136 u64 data)
3137{
3138 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3139
3140 return dd->send_egress_err_status_cnt[56];
3141}
3142
3143static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3144 void *context, int vl, int mode,
3145 u64 data)
3146{
3147 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3148
3149 return dd->send_egress_err_status_cnt[55];
3150}
3151
3152static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3153 void *context, int vl, int mode,
3154 u64 data)
3155{
3156 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3157
3158 return dd->send_egress_err_status_cnt[54];
3159}
3160
3161static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3162 void *context, int vl, int mode,
3163 u64 data)
3164{
3165 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3166
3167 return dd->send_egress_err_status_cnt[53];
3168}
3169
3170static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3171 void *context, int vl, int mode,
3172 u64 data)
3173{
3174 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3175
3176 return dd->send_egress_err_status_cnt[52];
3177}
3178
3179static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3180 void *context, int vl, int mode,
3181 u64 data)
3182{
3183 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3184
3185 return dd->send_egress_err_status_cnt[51];
3186}
3187
3188static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3189 void *context, int vl, int mode,
3190 u64 data)
3191{
3192 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3193
3194 return dd->send_egress_err_status_cnt[50];
3195}
3196
3197static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3198 void *context, int vl, int mode,
3199 u64 data)
3200{
3201 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3202
3203 return dd->send_egress_err_status_cnt[49];
3204}
3205
3206static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3207 void *context, int vl, int mode,
3208 u64 data)
3209{
3210 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3211
3212 return dd->send_egress_err_status_cnt[48];
3213}
3214
3215static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3216 void *context, int vl, int mode,
3217 u64 data)
3218{
3219 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3220
3221 return dd->send_egress_err_status_cnt[47];
3222}
3223
3224static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3225 void *context, int vl, int mode,
3226 u64 data)
3227{
3228 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3229
3230 return dd->send_egress_err_status_cnt[46];
3231}
3232
3233static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3234 void *context, int vl, int mode,
3235 u64 data)
3236{
3237 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3238
3239 return dd->send_egress_err_status_cnt[45];
3240}
3241
3242static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3243 void *context, int vl,
3244 int mode, u64 data)
3245{
3246 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3247
3248 return dd->send_egress_err_status_cnt[44];
3249}
3250
3251static u64 access_tx_read_sdma_memory_unc_err_cnt(
3252 const struct cntr_entry *entry,
3253 void *context, int vl, int mode, u64 data)
3254{
3255 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3256
3257 return dd->send_egress_err_status_cnt[43];
3258}
3259
3260static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3261 void *context, int vl, int mode,
3262 u64 data)
3263{
3264 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3265
3266 return dd->send_egress_err_status_cnt[42];
3267}
3268
3269static u64 access_tx_credit_return_partiy_err_cnt(
3270 const struct cntr_entry *entry,
3271 void *context, int vl, int mode, u64 data)
3272{
3273 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3274
3275 return dd->send_egress_err_status_cnt[41];
3276}
3277
3278static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3279 const struct cntr_entry *entry,
3280 void *context, int vl, int mode, u64 data)
3281{
3282 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3283
3284 return dd->send_egress_err_status_cnt[40];
3285}
3286
3287static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3288 const struct cntr_entry *entry,
3289 void *context, int vl, int mode, u64 data)
3290{
3291 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3292
3293 return dd->send_egress_err_status_cnt[39];
3294}
3295
3296static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3297 const struct cntr_entry *entry,
3298 void *context, int vl, int mode, u64 data)
3299{
3300 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3301
3302 return dd->send_egress_err_status_cnt[38];
3303}
3304
3305static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3306 const struct cntr_entry *entry,
3307 void *context, int vl, int mode, u64 data)
3308{
3309 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3310
3311 return dd->send_egress_err_status_cnt[37];
3312}
3313
3314static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3315 const struct cntr_entry *entry,
3316 void *context, int vl, int mode, u64 data)
3317{
3318 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3319
3320 return dd->send_egress_err_status_cnt[36];
3321}
3322
3323static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3324 const struct cntr_entry *entry,
3325 void *context, int vl, int mode, u64 data)
3326{
3327 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3328
3329 return dd->send_egress_err_status_cnt[35];
3330}
3331
3332static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3333 const struct cntr_entry *entry,
3334 void *context, int vl, int mode, u64 data)
3335{
3336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3337
3338 return dd->send_egress_err_status_cnt[34];
3339}
3340
3341static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3342 const struct cntr_entry *entry,
3343 void *context, int vl, int mode, u64 data)
3344{
3345 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3346
3347 return dd->send_egress_err_status_cnt[33];
3348}
3349
3350static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3351 const struct cntr_entry *entry,
3352 void *context, int vl, int mode, u64 data)
3353{
3354 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3355
3356 return dd->send_egress_err_status_cnt[32];
3357}
3358
3359static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3360 const struct cntr_entry *entry,
3361 void *context, int vl, int mode, u64 data)
3362{
3363 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3364
3365 return dd->send_egress_err_status_cnt[31];
3366}
3367
3368static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3369 const struct cntr_entry *entry,
3370 void *context, int vl, int mode, u64 data)
3371{
3372 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3373
3374 return dd->send_egress_err_status_cnt[30];
3375}
3376
3377static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3378 const struct cntr_entry *entry,
3379 void *context, int vl, int mode, u64 data)
3380{
3381 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3382
3383 return dd->send_egress_err_status_cnt[29];
3384}
3385
3386static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3387 const struct cntr_entry *entry,
3388 void *context, int vl, int mode, u64 data)
3389{
3390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3391
3392 return dd->send_egress_err_status_cnt[28];
3393}
3394
3395static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3396 const struct cntr_entry *entry,
3397 void *context, int vl, int mode, u64 data)
3398{
3399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3400
3401 return dd->send_egress_err_status_cnt[27];
3402}
3403
3404static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3405 const struct cntr_entry *entry,
3406 void *context, int vl, int mode, u64 data)
3407{
3408 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3409
3410 return dd->send_egress_err_status_cnt[26];
3411}
3412
3413static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3414 const struct cntr_entry *entry,
3415 void *context, int vl, int mode, u64 data)
3416{
3417 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3418
3419 return dd->send_egress_err_status_cnt[25];
3420}
3421
3422static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3423 const struct cntr_entry *entry,
3424 void *context, int vl, int mode, u64 data)
3425{
3426 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3427
3428 return dd->send_egress_err_status_cnt[24];
3429}
3430
3431static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3432 const struct cntr_entry *entry,
3433 void *context, int vl, int mode, u64 data)
3434{
3435 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3436
3437 return dd->send_egress_err_status_cnt[23];
3438}
3439
3440static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3441 const struct cntr_entry *entry,
3442 void *context, int vl, int mode, u64 data)
3443{
3444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3445
3446 return dd->send_egress_err_status_cnt[22];
3447}
3448
3449static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3450 const struct cntr_entry *entry,
3451 void *context, int vl, int mode, u64 data)
3452{
3453 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3454
3455 return dd->send_egress_err_status_cnt[21];
3456}
3457
3458static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3459 const struct cntr_entry *entry,
3460 void *context, int vl, int mode, u64 data)
3461{
3462 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3463
3464 return dd->send_egress_err_status_cnt[20];
3465}
3466
3467static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3468 const struct cntr_entry *entry,
3469 void *context, int vl, int mode, u64 data)
3470{
3471 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3472
3473 return dd->send_egress_err_status_cnt[19];
3474}
3475
3476static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3477 const struct cntr_entry *entry,
3478 void *context, int vl, int mode, u64 data)
3479{
3480 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3481
3482 return dd->send_egress_err_status_cnt[18];
3483}
3484
3485static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3486 const struct cntr_entry *entry,
3487 void *context, int vl, int mode, u64 data)
3488{
3489 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3490
3491 return dd->send_egress_err_status_cnt[17];
3492}
3493
3494static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3495 const struct cntr_entry *entry,
3496 void *context, int vl, int mode, u64 data)
3497{
3498 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3499
3500 return dd->send_egress_err_status_cnt[16];
3501}
3502
3503static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3504 void *context, int vl, int mode,
3505 u64 data)
3506{
3507 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3508
3509 return dd->send_egress_err_status_cnt[15];
3510}
3511
3512static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3513 void *context, int vl,
3514 int mode, u64 data)
3515{
3516 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3517
3518 return dd->send_egress_err_status_cnt[14];
3519}
3520
3521static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3522 void *context, int vl, int mode,
3523 u64 data)
3524{
3525 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3526
3527 return dd->send_egress_err_status_cnt[13];
3528}
3529
3530static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3531 void *context, int vl, int mode,
3532 u64 data)
3533{
3534 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3535
3536 return dd->send_egress_err_status_cnt[12];
3537}
3538
3539static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3540 const struct cntr_entry *entry,
3541 void *context, int vl, int mode, u64 data)
3542{
3543 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3544
3545 return dd->send_egress_err_status_cnt[11];
3546}
3547
3548static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3549 void *context, int vl, int mode,
3550 u64 data)
3551{
3552 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3553
3554 return dd->send_egress_err_status_cnt[10];
3555}
3556
3557static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3558 void *context, int vl, int mode,
3559 u64 data)
3560{
3561 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3562
3563 return dd->send_egress_err_status_cnt[9];
3564}
3565
3566static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3567 const struct cntr_entry *entry,
3568 void *context, int vl, int mode, u64 data)
3569{
3570 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3571
3572 return dd->send_egress_err_status_cnt[8];
3573}
3574
3575static u64 access_tx_pio_launch_intf_parity_err_cnt(
3576 const struct cntr_entry *entry,
3577 void *context, int vl, int mode, u64 data)
3578{
3579 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3580
3581 return dd->send_egress_err_status_cnt[7];
3582}
3583
3584static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3585 void *context, int vl, int mode,
3586 u64 data)
3587{
3588 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3589
3590 return dd->send_egress_err_status_cnt[6];
3591}
3592
3593static u64 access_tx_incorrect_link_state_err_cnt(
3594 const struct cntr_entry *entry,
3595 void *context, int vl, int mode, u64 data)
3596{
3597 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3598
3599 return dd->send_egress_err_status_cnt[5];
3600}
3601
3602static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3603 void *context, int vl, int mode,
3604 u64 data)
3605{
3606 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3607
3608 return dd->send_egress_err_status_cnt[4];
3609}
3610
3611static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3612 const struct cntr_entry *entry,
3613 void *context, int vl, int mode, u64 data)
3614{
3615 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3616
3617 return dd->send_egress_err_status_cnt[3];
3618}
3619
3620static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3621 void *context, int vl, int mode,
3622 u64 data)
3623{
3624 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3625
3626 return dd->send_egress_err_status_cnt[2];
3627}
3628
3629static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3630 const struct cntr_entry *entry,
3631 void *context, int vl, int mode, u64 data)
3632{
3633 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3634
3635 return dd->send_egress_err_status_cnt[1];
3636}
3637
3638static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3639 const struct cntr_entry *entry,
3640 void *context, int vl, int mode, u64 data)
3641{
3642 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3643
3644 return dd->send_egress_err_status_cnt[0];
3645}
3646
3647/*
3648 * Software counters corresponding to each of the
3649 * error status bits within SendErrStatus
3650 */
3651static u64 access_send_csr_write_bad_addr_err_cnt(
3652 const struct cntr_entry *entry,
3653 void *context, int vl, int mode, u64 data)
3654{
3655 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3656
3657 return dd->send_err_status_cnt[2];
3658}
3659
3660static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3661 void *context, int vl,
3662 int mode, u64 data)
3663{
3664 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3665
3666 return dd->send_err_status_cnt[1];
3667}
3668
3669static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3670 void *context, int vl, int mode,
3671 u64 data)
3672{
3673 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3674
3675 return dd->send_err_status_cnt[0];
3676}
3677
3678/*
3679 * Software counters corresponding to each of the
3680 * error status bits within SendCtxtErrStatus
3681 */
3682static u64 access_pio_write_out_of_bounds_err_cnt(
3683 const struct cntr_entry *entry,
3684 void *context, int vl, int mode, u64 data)
3685{
3686 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3687
3688 return dd->sw_ctxt_err_status_cnt[4];
3689}
3690
3691static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3692 void *context, int vl, int mode,
3693 u64 data)
3694{
3695 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3696
3697 return dd->sw_ctxt_err_status_cnt[3];
3698}
3699
3700static u64 access_pio_write_crosses_boundary_err_cnt(
3701 const struct cntr_entry *entry,
3702 void *context, int vl, int mode, u64 data)
3703{
3704 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3705
3706 return dd->sw_ctxt_err_status_cnt[2];
3707}
3708
3709static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3710 void *context, int vl,
3711 int mode, u64 data)
3712{
3713 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3714
3715 return dd->sw_ctxt_err_status_cnt[1];
3716}
3717
3718static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3719 void *context, int vl, int mode,
3720 u64 data)
3721{
3722 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3723
3724 return dd->sw_ctxt_err_status_cnt[0];
3725}
3726
3727/*
3728 * Software counters corresponding to each of the
3729 * error status bits within SendDmaEngErrStatus
3730 */
3731static u64 access_sdma_header_request_fifo_cor_err_cnt(
3732 const struct cntr_entry *entry,
3733 void *context, int vl, int mode, u64 data)
3734{
3735 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3736
3737 return dd->sw_send_dma_eng_err_status_cnt[23];
3738}
3739
3740static u64 access_sdma_header_storage_cor_err_cnt(
3741 const struct cntr_entry *entry,
3742 void *context, int vl, int mode, u64 data)
3743{
3744 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3745
3746 return dd->sw_send_dma_eng_err_status_cnt[22];
3747}
3748
3749static u64 access_sdma_packet_tracking_cor_err_cnt(
3750 const struct cntr_entry *entry,
3751 void *context, int vl, int mode, u64 data)
3752{
3753 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3754
3755 return dd->sw_send_dma_eng_err_status_cnt[21];
3756}
3757
3758static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3759 void *context, int vl, int mode,
3760 u64 data)
3761{
3762 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3763
3764 return dd->sw_send_dma_eng_err_status_cnt[20];
3765}
3766
3767static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3768 void *context, int vl, int mode,
3769 u64 data)
3770{
3771 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3772
3773 return dd->sw_send_dma_eng_err_status_cnt[19];
3774}
3775
3776static u64 access_sdma_header_request_fifo_unc_err_cnt(
3777 const struct cntr_entry *entry,
3778 void *context, int vl, int mode, u64 data)
3779{
3780 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3781
3782 return dd->sw_send_dma_eng_err_status_cnt[18];
3783}
3784
3785static u64 access_sdma_header_storage_unc_err_cnt(
3786 const struct cntr_entry *entry,
3787 void *context, int vl, int mode, u64 data)
3788{
3789 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3790
3791 return dd->sw_send_dma_eng_err_status_cnt[17];
3792}
3793
3794static u64 access_sdma_packet_tracking_unc_err_cnt(
3795 const struct cntr_entry *entry,
3796 void *context, int vl, int mode, u64 data)
3797{
3798 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3799
3800 return dd->sw_send_dma_eng_err_status_cnt[16];
3801}
3802
3803static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3804 void *context, int vl, int mode,
3805 u64 data)
3806{
3807 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3808
3809 return dd->sw_send_dma_eng_err_status_cnt[15];
3810}
3811
3812static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3813 void *context, int vl, int mode,
3814 u64 data)
3815{
3816 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3817
3818 return dd->sw_send_dma_eng_err_status_cnt[14];
3819}
3820
3821static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3822 void *context, int vl, int mode,
3823 u64 data)
3824{
3825 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3826
3827 return dd->sw_send_dma_eng_err_status_cnt[13];
3828}
3829
3830static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3831 void *context, int vl, int mode,
3832 u64 data)
3833{
3834 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3835
3836 return dd->sw_send_dma_eng_err_status_cnt[12];
3837}
3838
3839static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3840 void *context, int vl, int mode,
3841 u64 data)
3842{
3843 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3844
3845 return dd->sw_send_dma_eng_err_status_cnt[11];
3846}
3847
3848static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3849 void *context, int vl, int mode,
3850 u64 data)
3851{
3852 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3853
3854 return dd->sw_send_dma_eng_err_status_cnt[10];
3855}
3856
3857static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3858 void *context, int vl, int mode,
3859 u64 data)
3860{
3861 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3862
3863 return dd->sw_send_dma_eng_err_status_cnt[9];
3864}
3865
3866static u64 access_sdma_packet_desc_overflow_err_cnt(
3867 const struct cntr_entry *entry,
3868 void *context, int vl, int mode, u64 data)
3869{
3870 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3871
3872 return dd->sw_send_dma_eng_err_status_cnt[8];
3873}
3874
3875static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3876 void *context, int vl,
3877 int mode, u64 data)
3878{
3879 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3880
3881 return dd->sw_send_dma_eng_err_status_cnt[7];
3882}
3883
3884static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3885 void *context, int vl, int mode, u64 data)
3886{
3887 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3888
3889 return dd->sw_send_dma_eng_err_status_cnt[6];
3890}
3891
3892static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3893 void *context, int vl, int mode,
3894 u64 data)
3895{
3896 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3897
3898 return dd->sw_send_dma_eng_err_status_cnt[5];
3899}
3900
3901static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3902 void *context, int vl, int mode,
3903 u64 data)
3904{
3905 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3906
3907 return dd->sw_send_dma_eng_err_status_cnt[4];
3908}
3909
3910static u64 access_sdma_tail_out_of_bounds_err_cnt(
3911 const struct cntr_entry *entry,
3912 void *context, int vl, int mode, u64 data)
3913{
3914 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3915
3916 return dd->sw_send_dma_eng_err_status_cnt[3];
3917}
3918
3919static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
3920 void *context, int vl, int mode,
3921 u64 data)
3922{
3923 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3924
3925 return dd->sw_send_dma_eng_err_status_cnt[2];
3926}
3927
3928static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
3929 void *context, int vl, int mode,
3930 u64 data)
3931{
3932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3933
3934 return dd->sw_send_dma_eng_err_status_cnt[1];
3935}
3936
3937static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
3938 void *context, int vl, int mode,
3939 u64 data)
3940{
3941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3942
3943 return dd->sw_send_dma_eng_err_status_cnt[0];
3944}
3945
77241056
MM		 3946#define def_access_sw_cpu(cntr) \
3947static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
3948 void *context, int vl, int mode, u64 data) \
3949{ \
3950 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4eb06882
DD		 3951 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \
3952 ppd->ibport_data.rvp.cntr, vl, \
77241056
MM		 3953 mode, data); \
3954}
3955
3956def_access_sw_cpu(rc_acks);
3957def_access_sw_cpu(rc_qacks);
3958def_access_sw_cpu(rc_delayed_comp);
3959
3960#define def_access_ibp_counter(cntr) \
3961static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
3962 void *context, int vl, int mode, u64 data) \
3963{ \
3964 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
3965 \
3966 if (vl != CNTR_INVALID_VL) \
3967 return 0; \
3968 \
4eb06882 3969 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \
77241056
MM		 3970 mode, data); \
3971}
3972
3973def_access_ibp_counter(loop_pkts);
3974def_access_ibp_counter(rc_resends);
3975def_access_ibp_counter(rnr_naks);
3976def_access_ibp_counter(other_naks);
3977def_access_ibp_counter(rc_timeouts);
3978def_access_ibp_counter(pkt_drops);
3979def_access_ibp_counter(dmawait);
3980def_access_ibp_counter(rc_seqnak);
3981def_access_ibp_counter(rc_dupreq);
3982def_access_ibp_counter(rdma_seq);
3983def_access_ibp_counter(unaligned);
3984def_access_ibp_counter(seq_naks);
3985
3986static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
3987[C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
3988[C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
3989 CNTR_NORMAL),
3990[C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
3991 CNTR_NORMAL),
3992[C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
3993 RCV_TID_FLOW_GEN_MISMATCH_CNT,
3994 CNTR_NORMAL),
77241056
MM		 3995[C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
3996 CNTR_NORMAL),
3997[C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
3998 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
3999[C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4000 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4001[C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4002 CNTR_NORMAL),
4003[C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4004 CNTR_NORMAL),
4005[C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4006 CNTR_NORMAL),
4007[C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4008 CNTR_NORMAL),
4009[C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4010 CNTR_NORMAL),
4011[C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4012 CNTR_NORMAL),
4013[C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4014 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4015[C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4016 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4017[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4018 CNTR_SYNTH),
4019[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
4020[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4021 CNTR_SYNTH),
4022[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4023 CNTR_SYNTH),
4024[C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4025 CNTR_SYNTH),
4026[C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4027 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4028[C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4029 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4030 CNTR_SYNTH),
4031[C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4032 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4033[C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4034 CNTR_SYNTH),
4035[C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4036 CNTR_SYNTH),
4037[C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4038 CNTR_SYNTH),
4039[C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4040 CNTR_SYNTH),
4041[C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4042 CNTR_SYNTH),
4043[C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4044 CNTR_SYNTH),
4045[C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4046 CNTR_SYNTH),
4047[C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4048 CNTR_SYNTH | CNTR_VL),
4049[C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4050 CNTR_SYNTH | CNTR_VL),
4051[C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4052[C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4053 CNTR_SYNTH | CNTR_VL),
4054[C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4055[C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4056 CNTR_SYNTH | CNTR_VL),
4057[C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4058 CNTR_SYNTH),
4059[C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4060 CNTR_SYNTH | CNTR_VL),
4061[C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4062 CNTR_SYNTH),
4063[C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4064 CNTR_SYNTH | CNTR_VL),
4065[C_DC_TOTAL_CRC] =
4066 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4067 CNTR_SYNTH),
4068[C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4069 CNTR_SYNTH),
4070[C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4071 CNTR_SYNTH),
4072[C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4073 CNTR_SYNTH),
4074[C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4075 CNTR_SYNTH),
4076[C_DC_CRC_MULT_LN] =
4077 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4078 CNTR_SYNTH),
4079[C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4080 CNTR_SYNTH),
4081[C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4082 CNTR_SYNTH),
4083[C_DC_SEQ_CRC_CNT] =
4084 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4085 CNTR_SYNTH),
4086[C_DC_ESC0_ONLY_CNT] =
4087 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4088 CNTR_SYNTH),
4089[C_DC_ESC0_PLUS1_CNT] =
4090 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4091 CNTR_SYNTH),
4092[C_DC_ESC0_PLUS2_CNT] =
4093 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4094 CNTR_SYNTH),
4095[C_DC_REINIT_FROM_PEER_CNT] =
4096 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4097 CNTR_SYNTH),
4098[C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4099 CNTR_SYNTH),
4100[C_DC_MISC_FLG_CNT] =
4101 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4102 CNTR_SYNTH),
4103[C_DC_PRF_GOOD_LTP_CNT] =
4104 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4105[C_DC_PRF_ACCEPTED_LTP_CNT] =
4106 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4107 CNTR_SYNTH),
4108[C_DC_PRF_RX_FLIT_CNT] =
4109 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4110[C_DC_PRF_TX_FLIT_CNT] =
4111 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4112[C_DC_PRF_CLK_CNTR] =
4113 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4114[C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4115 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4116[C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4117 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4118 CNTR_SYNTH),
4119[C_DC_PG_STS_TX_SBE_CNT] =
4120 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4121[C_DC_PG_STS_TX_MBE_CNT] =
4122 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4123 CNTR_SYNTH),
4124[C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4125 access_sw_cpu_intr),
4126[C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4127 access_sw_cpu_rcv_limit),
4128[C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4129 access_sw_vtx_wait),
4130[C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4131 access_sw_pio_wait),
14553ca1
MM		 4132[C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4133 access_sw_pio_drain),
77241056
MM		 4134[C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4135 access_sw_kmem_wait),
b421922e
DL		 4136[C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4137 access_sw_send_schedule),
a699c6c2
VM		 4138[C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4139 SEND_DMA_DESC_FETCHED_CNT, 0,
4140 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4141 dev_access_u32_csr),
4142[C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4143 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4144 access_sde_int_cnt),
4145[C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4146 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4147 access_sde_err_cnt),
4148[C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4149 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4150 access_sde_idle_int_cnt),
4151[C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4152 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4153 access_sde_progress_int_cnt),
2c5b521a
JR		 4154/* MISC_ERR_STATUS */
4155[C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4156 CNTR_NORMAL,
4157 access_misc_pll_lock_fail_err_cnt),
4158[C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4159 CNTR_NORMAL,
4160 access_misc_mbist_fail_err_cnt),
4161[C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4162 CNTR_NORMAL,
4163 access_misc_invalid_eep_cmd_err_cnt),
4164[C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4165 CNTR_NORMAL,
4166 access_misc_efuse_done_parity_err_cnt),
4167[C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4168 CNTR_NORMAL,
4169 access_misc_efuse_write_err_cnt),
4170[C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4171 0, CNTR_NORMAL,
4172 access_misc_efuse_read_bad_addr_err_cnt),
4173[C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4174 CNTR_NORMAL,
4175 access_misc_efuse_csr_parity_err_cnt),
4176[C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4177 CNTR_NORMAL,
4178 access_misc_fw_auth_failed_err_cnt),
4179[C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4180 CNTR_NORMAL,
4181 access_misc_key_mismatch_err_cnt),
4182[C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4183 CNTR_NORMAL,
4184 access_misc_sbus_write_failed_err_cnt),
4185[C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4186 CNTR_NORMAL,
4187 access_misc_csr_write_bad_addr_err_cnt),
4188[C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4189 CNTR_NORMAL,
4190 access_misc_csr_read_bad_addr_err_cnt),
4191[C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4192 CNTR_NORMAL,
4193 access_misc_csr_parity_err_cnt),
4194/* CceErrStatus */
4195[C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4196 CNTR_NORMAL,
4197 access_sw_cce_err_status_aggregated_cnt),
4198[C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4199 CNTR_NORMAL,
4200 access_cce_msix_csr_parity_err_cnt),
4201[C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4202 CNTR_NORMAL,
4203 access_cce_int_map_unc_err_cnt),
4204[C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4205 CNTR_NORMAL,
4206 access_cce_int_map_cor_err_cnt),
4207[C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4208 CNTR_NORMAL,
4209 access_cce_msix_table_unc_err_cnt),
4210[C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4211 CNTR_NORMAL,
4212 access_cce_msix_table_cor_err_cnt),
4213[C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4214 0, CNTR_NORMAL,
4215 access_cce_rxdma_conv_fifo_parity_err_cnt),
4216[C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4217 0, CNTR_NORMAL,
4218 access_cce_rcpl_async_fifo_parity_err_cnt),
4219[C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4220 CNTR_NORMAL,
4221 access_cce_seg_write_bad_addr_err_cnt),
4222[C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4223 CNTR_NORMAL,
4224 access_cce_seg_read_bad_addr_err_cnt),
4225[C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4226 CNTR_NORMAL,
4227 access_la_triggered_cnt),
4228[C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4229 CNTR_NORMAL,
4230 access_cce_trgt_cpl_timeout_err_cnt),
4231[C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4232 CNTR_NORMAL,
4233 access_pcic_receive_parity_err_cnt),
4234[C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4235 CNTR_NORMAL,
4236 access_pcic_transmit_back_parity_err_cnt),
4237[C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4238 0, CNTR_NORMAL,
4239 access_pcic_transmit_front_parity_err_cnt),
4240[C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4241 CNTR_NORMAL,
4242 access_pcic_cpl_dat_q_unc_err_cnt),
4243[C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4244 CNTR_NORMAL,
4245 access_pcic_cpl_hd_q_unc_err_cnt),
4246[C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4247 CNTR_NORMAL,
4248 access_pcic_post_dat_q_unc_err_cnt),
4249[C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4250 CNTR_NORMAL,
4251 access_pcic_post_hd_q_unc_err_cnt),
4252[C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4253 CNTR_NORMAL,
4254 access_pcic_retry_sot_mem_unc_err_cnt),
4255[C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4256 CNTR_NORMAL,
4257 access_pcic_retry_mem_unc_err),
4258[C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4259 CNTR_NORMAL,
4260 access_pcic_n_post_dat_q_parity_err_cnt),
4261[C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4262 CNTR_NORMAL,
4263 access_pcic_n_post_h_q_parity_err_cnt),
4264[C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4265 CNTR_NORMAL,
4266 access_pcic_cpl_dat_q_cor_err_cnt),
4267[C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4268 CNTR_NORMAL,
4269 access_pcic_cpl_hd_q_cor_err_cnt),
4270[C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4271 CNTR_NORMAL,
4272 access_pcic_post_dat_q_cor_err_cnt),
4273[C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4274 CNTR_NORMAL,
4275 access_pcic_post_hd_q_cor_err_cnt),
4276[C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4277 CNTR_NORMAL,
4278 access_pcic_retry_sot_mem_cor_err_cnt),
4279[C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4280 CNTR_NORMAL,
4281 access_pcic_retry_mem_cor_err_cnt),
4282[C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4283 "CceCli1AsyncFifoDbgParityError", 0, 0,
4284 CNTR_NORMAL,
4285 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4286[C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4287 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4288 CNTR_NORMAL,
4289 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4290 ),
4291[C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4292 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4293 CNTR_NORMAL,
4294 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4295[C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4296 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4297 CNTR_NORMAL,
4298 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4299[C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4300 0, CNTR_NORMAL,
4301 access_cce_cli2_async_fifo_parity_err_cnt),
4302[C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4303 CNTR_NORMAL,
4304 access_cce_csr_cfg_bus_parity_err_cnt),
4305[C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4306 0, CNTR_NORMAL,
4307 access_cce_cli0_async_fifo_parity_err_cnt),
4308[C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4309 CNTR_NORMAL,
4310 access_cce_rspd_data_parity_err_cnt),
4311[C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4312 CNTR_NORMAL,
4313 access_cce_trgt_access_err_cnt),
4314[C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4315 0, CNTR_NORMAL,
4316 access_cce_trgt_async_fifo_parity_err_cnt),
4317[C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4318 CNTR_NORMAL,
4319 access_cce_csr_write_bad_addr_err_cnt),
4320[C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4321 CNTR_NORMAL,
4322 access_cce_csr_read_bad_addr_err_cnt),
4323[C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4324 CNTR_NORMAL,
4325 access_ccs_csr_parity_err_cnt),
4326
4327/* RcvErrStatus */
4328[C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4329 CNTR_NORMAL,
4330 access_rx_csr_parity_err_cnt),
4331[C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4332 CNTR_NORMAL,
4333 access_rx_csr_write_bad_addr_err_cnt),
4334[C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4335 CNTR_NORMAL,
4336 access_rx_csr_read_bad_addr_err_cnt),
4337[C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4338 CNTR_NORMAL,
4339 access_rx_dma_csr_unc_err_cnt),
4340[C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4341 CNTR_NORMAL,
4342 access_rx_dma_dq_fsm_encoding_err_cnt),
4343[C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4344 CNTR_NORMAL,
4345 access_rx_dma_eq_fsm_encoding_err_cnt),
4346[C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4347 CNTR_NORMAL,
4348 access_rx_dma_csr_parity_err_cnt),
4349[C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4350 CNTR_NORMAL,
4351 access_rx_rbuf_data_cor_err_cnt),
4352[C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4353 CNTR_NORMAL,
4354 access_rx_rbuf_data_unc_err_cnt),
4355[C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4356 CNTR_NORMAL,
4357 access_rx_dma_data_fifo_rd_cor_err_cnt),
4358[C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4359 CNTR_NORMAL,
4360 access_rx_dma_data_fifo_rd_unc_err_cnt),
4361[C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4362 CNTR_NORMAL,
4363 access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4364[C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4365 CNTR_NORMAL,
4366 access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4367[C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4368 CNTR_NORMAL,
4369 access_rx_rbuf_desc_part2_cor_err_cnt),
4370[C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4371 CNTR_NORMAL,
4372 access_rx_rbuf_desc_part2_unc_err_cnt),
4373[C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4374 CNTR_NORMAL,
4375 access_rx_rbuf_desc_part1_cor_err_cnt),
4376[C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4377 CNTR_NORMAL,
4378 access_rx_rbuf_desc_part1_unc_err_cnt),
4379[C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4380 CNTR_NORMAL,
4381 access_rx_hq_intr_fsm_err_cnt),
4382[C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4383 CNTR_NORMAL,
4384 access_rx_hq_intr_csr_parity_err_cnt),
4385[C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4386 CNTR_NORMAL,
4387 access_rx_lookup_csr_parity_err_cnt),
4388[C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4389 CNTR_NORMAL,
4390 access_rx_lookup_rcv_array_cor_err_cnt),
4391[C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4392 CNTR_NORMAL,
4393 access_rx_lookup_rcv_array_unc_err_cnt),
4394[C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4395 0, CNTR_NORMAL,
4396 access_rx_lookup_des_part2_parity_err_cnt),
4397[C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4398 0, CNTR_NORMAL,
4399 access_rx_lookup_des_part1_unc_cor_err_cnt),
4400[C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4401 CNTR_NORMAL,
4402 access_rx_lookup_des_part1_unc_err_cnt),
4403[C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4404 CNTR_NORMAL,
4405 access_rx_rbuf_next_free_buf_cor_err_cnt),
4406[C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4407 CNTR_NORMAL,
4408 access_rx_rbuf_next_free_buf_unc_err_cnt),
4409[C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4410 "RxRbufFlInitWrAddrParityErr", 0, 0,
4411 CNTR_NORMAL,
4412 access_rbuf_fl_init_wr_addr_parity_err_cnt),
4413[C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4414 0, CNTR_NORMAL,
4415 access_rx_rbuf_fl_initdone_parity_err_cnt),
4416[C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4417 0, CNTR_NORMAL,
4418 access_rx_rbuf_fl_write_addr_parity_err_cnt),
4419[C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4420 CNTR_NORMAL,
4421 access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4422[C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4423 CNTR_NORMAL,
4424 access_rx_rbuf_empty_err_cnt),
4425[C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4426 CNTR_NORMAL,
4427 access_rx_rbuf_full_err_cnt),
4428[C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4429 CNTR_NORMAL,
4430 access_rbuf_bad_lookup_err_cnt),
4431[C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4432 CNTR_NORMAL,
4433 access_rbuf_ctx_id_parity_err_cnt),
4434[C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4435 CNTR_NORMAL,
4436 access_rbuf_csr_qeopdw_parity_err_cnt),
4437[C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4438 "RxRbufCsrQNumOfPktParityErr", 0, 0,
4439 CNTR_NORMAL,
4440 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4441[C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4442 "RxRbufCsrQTlPtrParityErr", 0, 0,
4443 CNTR_NORMAL,
4444 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4445[C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4446 0, CNTR_NORMAL,
4447 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4448[C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4449 0, CNTR_NORMAL,
4450 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4451[C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4452 0, 0, CNTR_NORMAL,
4453 access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4454[C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4455 0, CNTR_NORMAL,
4456 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4457[C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4458 "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4459 CNTR_NORMAL,
4460 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4461[C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4462 0, CNTR_NORMAL,
4463 access_rx_rbuf_block_list_read_cor_err_cnt),
4464[C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4465 0, CNTR_NORMAL,
4466 access_rx_rbuf_block_list_read_unc_err_cnt),
4467[C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4468 CNTR_NORMAL,
4469 access_rx_rbuf_lookup_des_cor_err_cnt),
4470[C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4471 CNTR_NORMAL,
4472 access_rx_rbuf_lookup_des_unc_err_cnt),
4473[C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4474 "RxRbufLookupDesRegUncCorErr", 0, 0,
4475 CNTR_NORMAL,
4476 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4477[C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4478 CNTR_NORMAL,
4479 access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4480[C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4481 CNTR_NORMAL,
4482 access_rx_rbuf_free_list_cor_err_cnt),
4483[C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4484 CNTR_NORMAL,
4485 access_rx_rbuf_free_list_unc_err_cnt),
4486[C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4487 CNTR_NORMAL,
4488 access_rx_rcv_fsm_encoding_err_cnt),
4489[C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4490 CNTR_NORMAL,
4491 access_rx_dma_flag_cor_err_cnt),
4492[C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4493 CNTR_NORMAL,
4494 access_rx_dma_flag_unc_err_cnt),
4495[C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4496 CNTR_NORMAL,
4497 access_rx_dc_sop_eop_parity_err_cnt),
4498[C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4499 CNTR_NORMAL,
4500 access_rx_rcv_csr_parity_err_cnt),
4501[C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4502 CNTR_NORMAL,
4503 access_rx_rcv_qp_map_table_cor_err_cnt),
4504[C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4505 CNTR_NORMAL,
4506 access_rx_rcv_qp_map_table_unc_err_cnt),
4507[C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4508 CNTR_NORMAL,
4509 access_rx_rcv_data_cor_err_cnt),
4510[C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4511 CNTR_NORMAL,
4512 access_rx_rcv_data_unc_err_cnt),
4513[C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4514 CNTR_NORMAL,
4515 access_rx_rcv_hdr_cor_err_cnt),
4516[C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4517 CNTR_NORMAL,
4518 access_rx_rcv_hdr_unc_err_cnt),
4519[C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4520 CNTR_NORMAL,
4521 access_rx_dc_intf_parity_err_cnt),
4522[C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4523 CNTR_NORMAL,
4524 access_rx_dma_csr_cor_err_cnt),
4525/* SendPioErrStatus */
4526[C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4527 CNTR_NORMAL,
4528 access_pio_pec_sop_head_parity_err_cnt),
4529[C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4530 CNTR_NORMAL,
4531 access_pio_pcc_sop_head_parity_err_cnt),
4532[C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4533 0, 0, CNTR_NORMAL,
4534 access_pio_last_returned_cnt_parity_err_cnt),
4535[C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4536 0, CNTR_NORMAL,
4537 access_pio_current_free_cnt_parity_err_cnt),
4538[C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4539 CNTR_NORMAL,
4540 access_pio_reserved_31_err_cnt),
4541[C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4542 CNTR_NORMAL,
4543 access_pio_reserved_30_err_cnt),
4544[C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4545 CNTR_NORMAL,
4546 access_pio_ppmc_sop_len_err_cnt),
4547[C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4548 CNTR_NORMAL,
4549 access_pio_ppmc_bqc_mem_parity_err_cnt),
4550[C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4551 CNTR_NORMAL,
4552 access_pio_vl_fifo_parity_err_cnt),
4553[C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4554 CNTR_NORMAL,
4555 access_pio_vlf_sop_parity_err_cnt),
4556[C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4557 CNTR_NORMAL,
4558 access_pio_vlf_v1_len_parity_err_cnt),
4559[C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4560 CNTR_NORMAL,
4561 access_pio_block_qw_count_parity_err_cnt),
4562[C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4563 CNTR_NORMAL,
4564 access_pio_write_qw_valid_parity_err_cnt),
4565[C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4566 CNTR_NORMAL,
4567 access_pio_state_machine_err_cnt),
4568[C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4569 CNTR_NORMAL,
4570 access_pio_write_data_parity_err_cnt),
4571[C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4572 CNTR_NORMAL,
4573 access_pio_host_addr_mem_cor_err_cnt),
4574[C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4575 CNTR_NORMAL,
4576 access_pio_host_addr_mem_unc_err_cnt),
4577[C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4578 CNTR_NORMAL,
4579 access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4580[C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4581 CNTR_NORMAL,
4582 access_pio_init_sm_in_err_cnt),
4583[C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4584 CNTR_NORMAL,
4585 access_pio_ppmc_pbl_fifo_err_cnt),
4586[C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4587 0, CNTR_NORMAL,
4588 access_pio_credit_ret_fifo_parity_err_cnt),
4589[C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4590 CNTR_NORMAL,
4591 access_pio_v1_len_mem_bank1_cor_err_cnt),
4592[C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4593 CNTR_NORMAL,
4594 access_pio_v1_len_mem_bank0_cor_err_cnt),
4595[C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4596 CNTR_NORMAL,
4597 access_pio_v1_len_mem_bank1_unc_err_cnt),
4598[C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4599 CNTR_NORMAL,
4600 access_pio_v1_len_mem_bank0_unc_err_cnt),
4601[C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4602 CNTR_NORMAL,
4603 access_pio_sm_pkt_reset_parity_err_cnt),
4604[C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4605 CNTR_NORMAL,
4606 access_pio_pkt_evict_fifo_parity_err_cnt),
4607[C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4608 "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4609 CNTR_NORMAL,
4610 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4611[C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4612 CNTR_NORMAL,
4613 access_pio_sbrdctl_crrel_parity_err_cnt),
4614[C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4615 CNTR_NORMAL,
4616 access_pio_pec_fifo_parity_err_cnt),
4617[C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4618 CNTR_NORMAL,
4619 access_pio_pcc_fifo_parity_err_cnt),
4620[C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4621 CNTR_NORMAL,
4622 access_pio_sb_mem_fifo1_err_cnt),
4623[C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4624 CNTR_NORMAL,
4625 access_pio_sb_mem_fifo0_err_cnt),
4626[C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4627 CNTR_NORMAL,
4628 access_pio_csr_parity_err_cnt),
4629[C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4630 CNTR_NORMAL,
4631 access_pio_write_addr_parity_err_cnt),
4632[C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4633 CNTR_NORMAL,
4634 access_pio_write_bad_ctxt_err_cnt),
4635/* SendDmaErrStatus */
4636[C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4637 0, CNTR_NORMAL,
4638 access_sdma_pcie_req_tracking_cor_err_cnt),
4639[C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4640 0, CNTR_NORMAL,
4641 access_sdma_pcie_req_tracking_unc_err_cnt),
4642[C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4643 CNTR_NORMAL,
4644 access_sdma_csr_parity_err_cnt),
4645[C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4646 CNTR_NORMAL,
4647 access_sdma_rpy_tag_err_cnt),
4648/* SendEgressErrStatus */
4649[C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4650 CNTR_NORMAL,
4651 access_tx_read_pio_memory_csr_unc_err_cnt),
4652[C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4653 0, CNTR_NORMAL,
4654 access_tx_read_sdma_memory_csr_err_cnt),
4655[C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4656 CNTR_NORMAL,
4657 access_tx_egress_fifo_cor_err_cnt),
4658[C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4659 CNTR_NORMAL,
4660 access_tx_read_pio_memory_cor_err_cnt),
4661[C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4662 CNTR_NORMAL,
4663 access_tx_read_sdma_memory_cor_err_cnt),
4664[C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4665 CNTR_NORMAL,
4666 access_tx_sb_hdr_cor_err_cnt),
4667[C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4668 CNTR_NORMAL,
4669 access_tx_credit_overrun_err_cnt),
4670[C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4671 CNTR_NORMAL,
4672 access_tx_launch_fifo8_cor_err_cnt),
4673[C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4674 CNTR_NORMAL,
4675 access_tx_launch_fifo7_cor_err_cnt),
4676[C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4677 CNTR_NORMAL,
4678 access_tx_launch_fifo6_cor_err_cnt),
4679[C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4680 CNTR_NORMAL,
4681 access_tx_launch_fifo5_cor_err_cnt),
4682[C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4683 CNTR_NORMAL,
4684 access_tx_launch_fifo4_cor_err_cnt),
4685[C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4686 CNTR_NORMAL,
4687 access_tx_launch_fifo3_cor_err_cnt),
4688[C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4689 CNTR_NORMAL,
4690 access_tx_launch_fifo2_cor_err_cnt),
4691[C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4692 CNTR_NORMAL,
4693 access_tx_launch_fifo1_cor_err_cnt),
4694[C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4695 CNTR_NORMAL,
4696 access_tx_launch_fifo0_cor_err_cnt),
4697[C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4698 CNTR_NORMAL,
4699 access_tx_credit_return_vl_err_cnt),
4700[C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4701 CNTR_NORMAL,
4702 access_tx_hcrc_insertion_err_cnt),
4703[C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4704 CNTR_NORMAL,
4705 access_tx_egress_fifo_unc_err_cnt),
4706[C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4707 CNTR_NORMAL,
4708 access_tx_read_pio_memory_unc_err_cnt),
4709[C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4710 CNTR_NORMAL,
4711 access_tx_read_sdma_memory_unc_err_cnt),
4712[C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4713 CNTR_NORMAL,
4714 access_tx_sb_hdr_unc_err_cnt),
4715[C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4716 CNTR_NORMAL,
4717 access_tx_credit_return_partiy_err_cnt),
4718[C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4719 0, 0, CNTR_NORMAL,
4720 access_tx_launch_fifo8_unc_or_parity_err_cnt),
4721[C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4722 0, 0, CNTR_NORMAL,
4723 access_tx_launch_fifo7_unc_or_parity_err_cnt),
4724[C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4725 0, 0, CNTR_NORMAL,
4726 access_tx_launch_fifo6_unc_or_parity_err_cnt),
4727[C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4728 0, 0, CNTR_NORMAL,
4729 access_tx_launch_fifo5_unc_or_parity_err_cnt),
4730[C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4731 0, 0, CNTR_NORMAL,
4732 access_tx_launch_fifo4_unc_or_parity_err_cnt),
4733[C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4734 0, 0, CNTR_NORMAL,
4735 access_tx_launch_fifo3_unc_or_parity_err_cnt),
4736[C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4737 0, 0, CNTR_NORMAL,
4738 access_tx_launch_fifo2_unc_or_parity_err_cnt),
4739[C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4740 0, 0, CNTR_NORMAL,
4741 access_tx_launch_fifo1_unc_or_parity_err_cnt),
4742[C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4743 0, 0, CNTR_NORMAL,
4744 access_tx_launch_fifo0_unc_or_parity_err_cnt),
4745[C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4746 0, 0, CNTR_NORMAL,
4747 access_tx_sdma15_disallowed_packet_err_cnt),
4748[C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4749 0, 0, CNTR_NORMAL,
4750 access_tx_sdma14_disallowed_packet_err_cnt),
4751[C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4752 0, 0, CNTR_NORMAL,
4753 access_tx_sdma13_disallowed_packet_err_cnt),
4754[C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4755 0, 0, CNTR_NORMAL,
4756 access_tx_sdma12_disallowed_packet_err_cnt),
4757[C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4758 0, 0, CNTR_NORMAL,
4759 access_tx_sdma11_disallowed_packet_err_cnt),
4760[C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4761 0, 0, CNTR_NORMAL,
4762 access_tx_sdma10_disallowed_packet_err_cnt),
4763[C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4764 0, 0, CNTR_NORMAL,
4765 access_tx_sdma9_disallowed_packet_err_cnt),
4766[C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4767 0, 0, CNTR_NORMAL,
4768 access_tx_sdma8_disallowed_packet_err_cnt),
4769[C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4770 0, 0, CNTR_NORMAL,
4771 access_tx_sdma7_disallowed_packet_err_cnt),
4772[C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4773 0, 0, CNTR_NORMAL,
4774 access_tx_sdma6_disallowed_packet_err_cnt),
4775[C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4776 0, 0, CNTR_NORMAL,
4777 access_tx_sdma5_disallowed_packet_err_cnt),
4778[C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4779 0, 0, CNTR_NORMAL,
4780 access_tx_sdma4_disallowed_packet_err_cnt),
4781[C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4782 0, 0, CNTR_NORMAL,
4783 access_tx_sdma3_disallowed_packet_err_cnt),
4784[C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4785 0, 0, CNTR_NORMAL,
4786 access_tx_sdma2_disallowed_packet_err_cnt),
4787[C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4788 0, 0, CNTR_NORMAL,
4789 access_tx_sdma1_disallowed_packet_err_cnt),
4790[C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4791 0, 0, CNTR_NORMAL,
4792 access_tx_sdma0_disallowed_packet_err_cnt),
4793[C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4794 CNTR_NORMAL,
4795 access_tx_config_parity_err_cnt),
4796[C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4797 CNTR_NORMAL,
4798 access_tx_sbrd_ctl_csr_parity_err_cnt),
4799[C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4800 CNTR_NORMAL,
4801 access_tx_launch_csr_parity_err_cnt),
4802[C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4803 CNTR_NORMAL,
4804 access_tx_illegal_vl_err_cnt),
4805[C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4806 "TxSbrdCtlStateMachineParityErr", 0, 0,
4807 CNTR_NORMAL,
4808 access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4809[C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4810 CNTR_NORMAL,
4811 access_egress_reserved_10_err_cnt),
4812[C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4813 CNTR_NORMAL,
4814 access_egress_reserved_9_err_cnt),
4815[C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4816 0, 0, CNTR_NORMAL,
4817 access_tx_sdma_launch_intf_parity_err_cnt),
4818[C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4819 CNTR_NORMAL,
4820 access_tx_pio_launch_intf_parity_err_cnt),
4821[C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4822 CNTR_NORMAL,
4823 access_egress_reserved_6_err_cnt),
4824[C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4825 CNTR_NORMAL,
4826 access_tx_incorrect_link_state_err_cnt),
4827[C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4828 CNTR_NORMAL,
4829 access_tx_linkdown_err_cnt),
4830[C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4831 "EgressFifoUnderrunOrParityErr", 0, 0,
4832 CNTR_NORMAL,
4833 access_tx_egress_fifi_underrun_or_parity_err_cnt),
4834[C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4835 CNTR_NORMAL,
4836 access_egress_reserved_2_err_cnt),
4837[C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4838 CNTR_NORMAL,
4839 access_tx_pkt_integrity_mem_unc_err_cnt),
4840[C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4841 CNTR_NORMAL,
4842 access_tx_pkt_integrity_mem_cor_err_cnt),
4843/* SendErrStatus */
4844[C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4845 CNTR_NORMAL,
4846 access_send_csr_write_bad_addr_err_cnt),
4847[C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4848 CNTR_NORMAL,
4849 access_send_csr_read_bad_addr_err_cnt),
4850[C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4851 CNTR_NORMAL,
4852 access_send_csr_parity_cnt),
4853/* SendCtxtErrStatus */
4854[C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4855 CNTR_NORMAL,
4856 access_pio_write_out_of_bounds_err_cnt),
4857[C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4858 CNTR_NORMAL,
4859 access_pio_write_overflow_err_cnt),
4860[C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4861 0, 0, CNTR_NORMAL,
4862 access_pio_write_crosses_boundary_err_cnt),
4863[C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4864 CNTR_NORMAL,
4865 access_pio_disallowed_packet_err_cnt),
4866[C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4867 CNTR_NORMAL,
4868 access_pio_inconsistent_sop_err_cnt),
4869/* SendDmaEngErrStatus */
4870[C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4871 0, 0, CNTR_NORMAL,
4872 access_sdma_header_request_fifo_cor_err_cnt),
4873[C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4874 CNTR_NORMAL,
4875 access_sdma_header_storage_cor_err_cnt),
4876[C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4877 CNTR_NORMAL,
4878 access_sdma_packet_tracking_cor_err_cnt),
4879[C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4880 CNTR_NORMAL,
4881 access_sdma_assembly_cor_err_cnt),
4882[C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4883 CNTR_NORMAL,
4884 access_sdma_desc_table_cor_err_cnt),
4885[C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4886 0, 0, CNTR_NORMAL,
4887 access_sdma_header_request_fifo_unc_err_cnt),
4888[C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
4889 CNTR_NORMAL,
4890 access_sdma_header_storage_unc_err_cnt),
4891[C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
4892 CNTR_NORMAL,
4893 access_sdma_packet_tracking_unc_err_cnt),
4894[C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
4895 CNTR_NORMAL,
4896 access_sdma_assembly_unc_err_cnt),
4897[C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
4898 CNTR_NORMAL,
4899 access_sdma_desc_table_unc_err_cnt),
4900[C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
4901 CNTR_NORMAL,
4902 access_sdma_timeout_err_cnt),
4903[C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
4904 CNTR_NORMAL,
4905 access_sdma_header_length_err_cnt),
4906[C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
4907 CNTR_NORMAL,
4908 access_sdma_header_address_err_cnt),
4909[C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
4910 CNTR_NORMAL,
4911 access_sdma_header_select_err_cnt),
4912[C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
4913 CNTR_NORMAL,
4914 access_sdma_reserved_9_err_cnt),
4915[C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
4916 CNTR_NORMAL,
4917 access_sdma_packet_desc_overflow_err_cnt),
4918[C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
4919 CNTR_NORMAL,
4920 access_sdma_length_mismatch_err_cnt),
4921[C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
4922 CNTR_NORMAL,
4923 access_sdma_halt_err_cnt),
4924[C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
4925 CNTR_NORMAL,
4926 access_sdma_mem_read_err_cnt),
4927[C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
4928 CNTR_NORMAL,
4929 access_sdma_first_desc_err_cnt),
4930[C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
4931 CNTR_NORMAL,
4932 access_sdma_tail_out_of_bounds_err_cnt),
4933[C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
4934 CNTR_NORMAL,
4935 access_sdma_too_long_err_cnt),
4936[C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
4937 CNTR_NORMAL,
4938 access_sdma_gen_mismatch_err_cnt),
4939[C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
4940 CNTR_NORMAL,
4941 access_sdma_wrong_dw_err_cnt),
77241056
MM		 4942};
4943
4944static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
4945[C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
4946 CNTR_NORMAL),
4947[C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
4948 CNTR_NORMAL),
4949[C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
4950 CNTR_NORMAL),
4951[C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
4952 CNTR_NORMAL),
4953[C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
4954 CNTR_NORMAL),
4955[C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
4956 CNTR_NORMAL),
4957[C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
4958 CNTR_NORMAL),
4959[C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
4960[C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
4961[C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
4962[C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
17fb4f29 4963 CNTR_SYNTH | CNTR_VL),
77241056 4964[C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
17fb4f29 4965 CNTR_SYNTH | CNTR_VL),
77241056 4966[C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
17fb4f29 4967 CNTR_SYNTH | CNTR_VL),
77241056
MM		 4968[C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
4969[C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
4970[C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4971 access_sw_link_dn_cnt),
77241056 4972[C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4973 access_sw_link_up_cnt),
6d014530
DL		 4974[C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
4975 access_sw_unknown_frame_cnt),
77241056 4976[C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4977 access_sw_xmit_discards),
77241056 4978[C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
17fb4f29
JJ		 4979 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
4980 access_sw_xmit_discards),
77241056 4981[C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
17fb4f29 4982 access_xmit_constraint_errs),
77241056 4983[C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
17fb4f29 4984 access_rcv_constraint_errs),
77241056
MM		 4985[C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
4986[C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
4987[C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
4988[C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
4989[C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
4990[C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
4991[C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
4992[C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
4993[C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
4994[C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
4995[C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
4996[C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
4997[C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
4998 access_sw_cpu_rc_acks),
4999[C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
17fb4f29 5000 access_sw_cpu_rc_qacks),
77241056 5001[C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
17fb4f29 5002 access_sw_cpu_rc_delayed_comp),
77241056
MM		 5003[OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5004[OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5005[OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5006[OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5007[OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5008[OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5009[OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5010[OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5011[OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5012[OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5013[OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5014[OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5015[OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5016[OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5017[OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5018[OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5019[OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5020[OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5021[OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5022[OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5023[OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5024[OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5025[OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5026[OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5027[OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5028[OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5029[OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5030[OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5031[OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5032[OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5033[OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5034[OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5035[OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5036[OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5037[OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5038[OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5039[OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5040[OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5041[OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5042[OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5043[OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5044[OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5045[OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5046[OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5047[OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5048[OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5049[OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5050[OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5051[OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5052[OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5053[OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5054[OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5055[OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5056[OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5057[OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5058[OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5059[OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5060[OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5061[OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5062[OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5063[OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5064[OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5065[OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5066[OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5067[OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5068[OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5069[OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5070[OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5071[OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5072[OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5073[OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5074[OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5075[OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5076[OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5077[OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5078[OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5079[OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5080[OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5081[OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5082[OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5083};
5084
5085/* ======================================================================== */
5086
77241056
MM		 5087/* return true if this is chip revision revision a */
5088int is_ax(struct hfi1_devdata *dd)
5089{
5090 u8 chip_rev_minor =
5091 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5092 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5093 return (chip_rev_minor & 0xf0) == 0;
5094}
5095
5096/* return true if this is chip revision revision b */
5097int is_bx(struct hfi1_devdata *dd)
5098{
5099 u8 chip_rev_minor =
5100 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5101 & CCE_REVISION_CHIP_REV_MINOR_MASK;
995deafa 5102 return (chip_rev_minor & 0xF0) == 0x10;
77241056
MM		 5103}
5104
5105/*
5106 * Append string s to buffer buf. Arguments curp and len are the current
5107 * position and remaining length, respectively.
5108 *
5109 * return 0 on success, 1 on out of room
5110 */
5111static int append_str(char *buf, char **curp, int *lenp, const char *s)
5112{
5113 char *p = *curp;
5114 int len = *lenp;
5115 int result = 0; /* success */
5116 char c;
5117
5118 /* add a comma, if first in the buffer */
5119 if (p != buf) {
5120 if (len == 0) {
5121 result = 1; /* out of room */
5122 goto done;
5123 }
5124 *p++ = ',';
5125 len--;
5126 }
5127
5128 /* copy the string */
5129 while ((c = *s++) != 0) {
5130 if (len == 0) {
5131 result = 1; /* out of room */
5132 goto done;
5133 }
5134 *p++ = c;
5135 len--;
5136 }
5137
5138done:
5139 /* write return values */
5140 *curp = p;
5141 *lenp = len;
5142
5143 return result;
5144}
5145
5146/*
5147 * Using the given flag table, print a comma separated string into
5148 * the buffer. End in '*' if the buffer is too short.
5149 */
5150static char *flag_string(char *buf, int buf_len, u64 flags,
17fb4f29 5151 struct flag_table *table, int table_size)
77241056
MM		 5152{
5153 char extra[32];
5154 char *p = buf;
5155 int len = buf_len;
5156 int no_room = 0;
5157 int i;
5158
5159 /* make sure there is at least 2 so we can form "*" */
5160 if (len < 2)
5161 return "";
5162
5163 len--; /* leave room for a nul */
5164 for (i = 0; i < table_size; i++) {
5165 if (flags & table[i].flag) {
5166 no_room = append_str(buf, &p, &len, table[i].str);
5167 if (no_room)
5168 break;
5169 flags &= ~table[i].flag;
5170 }
5171 }
5172
5173 /* any undocumented bits left? */
5174 if (!no_room && flags) {
5175 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5176 no_room = append_str(buf, &p, &len, extra);
5177 }
5178
5179 /* add * if ran out of room */
5180 if (no_room) {
5181 /* may need to back up to add space for a '*' */
5182 if (len == 0)
5183 --p;
5184 *p++ = '*';
5185 }
5186
5187 /* add final nul - space already allocated above */
5188 *p = 0;
5189 return buf;
5190}
5191
5192/* first 8 CCE error interrupt source names */
5193static const char * const cce_misc_names[] = {
5194 "CceErrInt", /* 0 */
5195 "RxeErrInt", /* 1 */
5196 "MiscErrInt", /* 2 */
5197 "Reserved3", /* 3 */
5198 "PioErrInt", /* 4 */
5199 "SDmaErrInt", /* 5 */
5200 "EgressErrInt", /* 6 */
5201 "TxeErrInt" /* 7 */
5202};
5203
5204/*
5205 * Return the miscellaneous error interrupt name.
5206 */
5207static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5208{
5209 if (source < ARRAY_SIZE(cce_misc_names))
5210 strncpy(buf, cce_misc_names[source], bsize);
5211 else
17fb4f29
JJ		 5212 snprintf(buf, bsize, "Reserved%u",
5213 source + IS_GENERAL_ERR_START);
77241056
MM		 5214
5215 return buf;
5216}
5217
5218/*
5219 * Return the SDMA engine error interrupt name.
5220 */
5221static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5222{
5223 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5224 return buf;
5225}
5226
5227/*
5228 * Return the send context error interrupt name.
5229 */
5230static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5231{
5232 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5233 return buf;
5234}
5235
5236static const char * const various_names[] = {
5237 "PbcInt",
5238 "GpioAssertInt",
5239 "Qsfp1Int",
5240 "Qsfp2Int",
5241 "TCritInt"
5242};
5243
5244/*
5245 * Return the various interrupt name.
5246 */
5247static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5248{
5249 if (source < ARRAY_SIZE(various_names))
5250 strncpy(buf, various_names[source], bsize);
5251 else
8638b77f 5252 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
77241056
MM		 5253 return buf;
5254}
5255
5256/*
5257 * Return the DC interrupt name.
5258 */
5259static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5260{
5261 static const char * const dc_int_names[] = {
5262 "common",
5263 "lcb",
5264 "8051",
5265 "lbm" /* local block merge */
5266 };
5267
5268 if (source < ARRAY_SIZE(dc_int_names))
5269 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5270 else
5271 snprintf(buf, bsize, "DCInt%u", source);
5272 return buf;
5273}
5274
5275static const char * const sdma_int_names[] = {
5276 "SDmaInt",
5277 "SdmaIdleInt",
5278 "SdmaProgressInt",
5279};
5280
5281/*
5282 * Return the SDMA engine interrupt name.
5283 */
5284static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5285{
5286 /* what interrupt */
5287 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
5288 /* which engine */
5289 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5290
5291 if (likely(what < 3))
5292 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5293 else
5294 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5295 return buf;
5296}
5297
5298/*
5299 * Return the receive available interrupt name.
5300 */
5301static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5302{
5303 snprintf(buf, bsize, "RcvAvailInt%u", source);
5304 return buf;
5305}
5306
5307/*
5308 * Return the receive urgent interrupt name.
5309 */
5310static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5311{
5312 snprintf(buf, bsize, "RcvUrgentInt%u", source);
5313 return buf;
5314}
5315
5316/*
5317 * Return the send credit interrupt name.
5318 */
5319static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5320{
5321 snprintf(buf, bsize, "SendCreditInt%u", source);
5322 return buf;
5323}
5324
5325/*
5326 * Return the reserved interrupt name.
5327 */
5328static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5329{
5330 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5331 return buf;
5332}
5333
5334static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5335{
5336 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5337 cce_err_status_flags,
5338 ARRAY_SIZE(cce_err_status_flags));
77241056
MM		 5339}
5340
5341static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5342{
5343 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5344 rxe_err_status_flags,
5345 ARRAY_SIZE(rxe_err_status_flags));
77241056
MM		 5346}
5347
5348static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5349{
5350 return flag_string(buf, buf_len, flags, misc_err_status_flags,
17fb4f29 5351 ARRAY_SIZE(misc_err_status_flags));
77241056
MM		 5352}
5353
5354static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5355{
5356 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5357 pio_err_status_flags,
5358 ARRAY_SIZE(pio_err_status_flags));
77241056
MM		 5359}
5360
5361static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5362{
5363 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5364 sdma_err_status_flags,
5365 ARRAY_SIZE(sdma_err_status_flags));
77241056
MM		 5366}
5367
5368static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5369{
5370 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5371 egress_err_status_flags,
5372 ARRAY_SIZE(egress_err_status_flags));
77241056
MM		 5373}
5374
5375static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5376{
5377 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5378 egress_err_info_flags,
5379 ARRAY_SIZE(egress_err_info_flags));
77241056
MM		 5380}
5381
5382static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5383{
5384 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5385 send_err_status_flags,
5386 ARRAY_SIZE(send_err_status_flags));
77241056
MM		 5387}
5388
5389static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5390{
5391 char buf[96];
2c5b521a 5392 int i = 0;
77241056
MM		 5393
5394 /*
5395 * For most these errors, there is nothing that can be done except
5396 * report or record it.
5397 */
5398 dd_dev_info(dd, "CCE Error: %s\n",
17fb4f29 5399 cce_err_status_string(buf, sizeof(buf), reg));
77241056 5400
995deafa
MM		 5401 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5402 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 5403 /* this error requires a manual drop into SPC freeze mode */
5404 /* then a fix up */
5405 start_freeze_handling(dd->pport, FREEZE_SELF);
5406 }
2c5b521a
JR		 5407
5408 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5409 if (reg & (1ull << i)) {
5410 incr_cntr64(&dd->cce_err_status_cnt[i]);
5411 /* maintain a counter over all cce_err_status errors */
5412 incr_cntr64(&dd->sw_cce_err_status_aggregate);
5413 }
5414 }
77241056
MM		 5415}
5416
5417/*
5418 * Check counters for receive errors that do not have an interrupt
5419 * associated with them.
5420 */
5421#define RCVERR_CHECK_TIME 10
5422static void update_rcverr_timer(unsigned long opaque)
5423{
5424 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
5425 struct hfi1_pportdata *ppd = dd->pport;
5426 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5427
5428 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
17fb4f29 5429 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
77241056 5430 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
17fb4f29
JJ		 5431 set_link_down_reason(
5432 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5433 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
77241056
MM		 5434 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
5435 }
50e5dcbe 5436 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
77241056
MM		 5437
5438 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5439}
5440
5441static int init_rcverr(struct hfi1_devdata *dd)
5442{
24523a94 5443 setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
77241056
MM		 5444 /* Assume the hardware counter has been reset */
5445 dd->rcv_ovfl_cnt = 0;
5446 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5447}
5448
5449static void free_rcverr(struct hfi1_devdata *dd)
5450{
5451 if (dd->rcverr_timer.data)
5452 del_timer_sync(&dd->rcverr_timer);
5453 dd->rcverr_timer.data = 0;
5454}
5455
5456static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5457{
5458 char buf[96];
2c5b521a 5459 int i = 0;
77241056
MM		 5460
5461 dd_dev_info(dd, "Receive Error: %s\n",
17fb4f29 5462 rxe_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5463
5464 if (reg & ALL_RXE_FREEZE_ERR) {
5465 int flags = 0;
5466
5467 /*
5468 * Freeze mode recovery is disabled for the errors
5469 * in RXE_FREEZE_ABORT_MASK
5470 */
995deafa 5471 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
77241056
MM		 5472 flags = FREEZE_ABORT;
5473
5474 start_freeze_handling(dd->pport, flags);
5475 }
2c5b521a
JR		 5476
5477 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5478 if (reg & (1ull << i))
5479 incr_cntr64(&dd->rcv_err_status_cnt[i]);
5480 }
77241056
MM		 5481}
5482
5483static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5484{
5485 char buf[96];
2c5b521a 5486 int i = 0;
77241056
MM		 5487
5488 dd_dev_info(dd, "Misc Error: %s",
17fb4f29 5489 misc_err_status_string(buf, sizeof(buf), reg));
2c5b521a
JR		 5490 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5491 if (reg & (1ull << i))
5492 incr_cntr64(&dd->misc_err_status_cnt[i]);
5493 }
77241056
MM		 5494}
5495
5496static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5497{
5498 char buf[96];
2c5b521a 5499 int i = 0;
77241056
MM		 5500
5501 dd_dev_info(dd, "PIO Error: %s\n",
17fb4f29 5502 pio_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5503
5504 if (reg & ALL_PIO_FREEZE_ERR)
5505 start_freeze_handling(dd->pport, 0);
2c5b521a
JR		 5506
5507 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5508 if (reg & (1ull << i))
5509 incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5510 }
77241056
MM		 5511}
5512
5513static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5514{
5515 char buf[96];
2c5b521a 5516 int i = 0;
77241056
MM		 5517
5518 dd_dev_info(dd, "SDMA Error: %s\n",
17fb4f29 5519 sdma_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5520
5521 if (reg & ALL_SDMA_FREEZE_ERR)
5522 start_freeze_handling(dd->pport, 0);
2c5b521a
JR		 5523
5524 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5525 if (reg & (1ull << i))
5526 incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5527 }
77241056
MM		 5528}
5529
69a00b8e 5530static inline void __count_port_discards(struct hfi1_pportdata *ppd)
77241056 5531{
69a00b8e
MM		 5532 incr_cntr64(&ppd->port_xmit_discards);
5533}
77241056 5534
69a00b8e
MM		 5535static void count_port_inactive(struct hfi1_devdata *dd)
5536{
5537 __count_port_discards(dd->pport);
77241056
MM		 5538}
5539
5540/*
5541 * We have had a "disallowed packet" error during egress. Determine the
5542 * integrity check which failed, and update relevant error counter, etc.
5543 *
5544 * Note that the SEND_EGRESS_ERR_INFO register has only a single
5545 * bit of state per integrity check, and so we can miss the reason for an
5546 * egress error if more than one packet fails the same integrity check
5547 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5548 */
69a00b8e
MM		 5549static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5550 int vl)
77241056
MM		 5551{
5552 struct hfi1_pportdata *ppd = dd->pport;
5553 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5554 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5555 char buf[96];
5556
5557 /* clear down all observed info as quickly as possible after read */
5558 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5559
5560 dd_dev_info(dd,
17fb4f29
JJ		 5561 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5562 info, egress_err_info_string(buf, sizeof(buf), info), src);
77241056
MM		 5563
5564 /* Eventually add other counters for each bit */
69a00b8e
MM		 5565 if (info & PORT_DISCARD_EGRESS_ERRS) {
5566 int weight, i;
77241056 5567
69a00b8e
MM		 5568 /*
5569 * Count all, in case multiple bits are set. Reminder:
5570 * since there is only one info register for many sources,
5571 * these may be attributed to the wrong VL if they occur
5572 * too close together.
5573 */
5574 weight = hweight64(info);
5575 for (i = 0; i < weight; i++) {
5576 __count_port_discards(ppd);
5577 if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5578 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5579 else if (vl == 15)
5580 incr_cntr64(&ppd->port_xmit_discards_vl
5581 [C_VL_15]);
5582 }
77241056
MM		 5583 }
5584}
5585
5586/*
5587 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5588 * register. Does it represent a 'port inactive' error?
5589 */
5590static inline int port_inactive_err(u64 posn)
5591{
5592 return (posn >= SEES(TX_LINKDOWN) &&
5593 posn <= SEES(TX_INCORRECT_LINK_STATE));
5594}
5595
5596/*
5597 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5598 * register. Does it represent a 'disallowed packet' error?
5599 */
69a00b8e 5600static inline int disallowed_pkt_err(int posn)
77241056
MM		 5601{
5602 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5603 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5604}
5605
69a00b8e
MM		 5606/*
5607 * Input value is a bit position of one of the SDMA engine disallowed
5608 * packet errors. Return which engine. Use of this must be guarded by
5609 * disallowed_pkt_err().
5610 */
5611static inline int disallowed_pkt_engine(int posn)
5612{
5613 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5614}
5615
5616/*
5617 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot
5618 * be done.
5619 */
5620static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5621{
5622 struct sdma_vl_map *m;
5623 int vl;
5624
5625 /* range check */
5626 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5627 return -1;
5628
5629 rcu_read_lock();
5630 m = rcu_dereference(dd->sdma_map);
5631 vl = m->engine_to_vl[engine];
5632 rcu_read_unlock();
5633
5634 return vl;
5635}
5636
5637/*
5638 * Translate the send context (sofware index) into a VL. Return -1 if the
5639 * translation cannot be done.
5640 */
5641static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5642{
5643 struct send_context_info *sci;
5644 struct send_context *sc;
5645 int i;
5646
5647 sci = &dd->send_contexts[sw_index];
5648
5649 /* there is no information for user (PSM) and ack contexts */
5650 if (sci->type != SC_KERNEL)
5651 return -1;
5652
5653 sc = sci->sc;
5654 if (!sc)
5655 return -1;
5656 if (dd->vld[15].sc == sc)
5657 return 15;
5658 for (i = 0; i < num_vls; i++)
5659 if (dd->vld[i].sc == sc)
5660 return i;
5661
5662 return -1;
5663}
5664
77241056
MM		 5665static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5666{
5667 u64 reg_copy = reg, handled = 0;
5668 char buf[96];
2c5b521a 5669 int i = 0;
77241056
MM		 5670
5671 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5672 start_freeze_handling(dd->pport, 0);
69a00b8e
MM		 5673 else if (is_ax(dd) &&
5674 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5675 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
77241056
MM		 5676 start_freeze_handling(dd->pport, 0);
5677
5678 while (reg_copy) {
5679 int posn = fls64(reg_copy);
69a00b8e 5680 /* fls64() returns a 1-based offset, we want it zero based */
77241056 5681 int shift = posn - 1;
69a00b8e 5682 u64 mask = 1ULL << shift;
77241056
MM		 5683
5684 if (port_inactive_err(shift)) {
5685 count_port_inactive(dd);
69a00b8e 5686 handled |= mask;
77241056 5687 } else if (disallowed_pkt_err(shift)) {
69a00b8e
MM		 5688 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5689
5690 handle_send_egress_err_info(dd, vl);
5691 handled |= mask;
77241056 5692 }
69a00b8e 5693 reg_copy &= ~mask;
77241056
MM		 5694 }
5695
5696 reg &= ~handled;
5697
5698 if (reg)
5699 dd_dev_info(dd, "Egress Error: %s\n",
17fb4f29 5700 egress_err_status_string(buf, sizeof(buf), reg));
2c5b521a
JR		 5701
5702 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5703 if (reg & (1ull << i))
5704 incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5705 }
77241056
MM		 5706}
5707
5708static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5709{
5710 char buf[96];
2c5b521a 5711 int i = 0;
77241056
MM		 5712
5713 dd_dev_info(dd, "Send Error: %s\n",
17fb4f29 5714 send_err_status_string(buf, sizeof(buf), reg));
77241056 5715
2c5b521a
JR		 5716 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5717 if (reg & (1ull << i))
5718 incr_cntr64(&dd->send_err_status_cnt[i]);
5719 }
77241056
MM		 5720}
5721
5722/*
5723 * The maximum number of times the error clear down will loop before
5724 * blocking a repeating error. This value is arbitrary.
5725 */
5726#define MAX_CLEAR_COUNT 20
5727
5728/*
5729 * Clear and handle an error register. All error interrupts are funneled
5730 * through here to have a central location to correctly handle single-
5731 * or multi-shot errors.
5732 *
5733 * For non per-context registers, call this routine with a context value
5734 * of 0 so the per-context offset is zero.
5735 *
5736 * If the handler loops too many times, assume that something is wrong
5737 * and can't be fixed, so mask the error bits.
5738 */
5739static void interrupt_clear_down(struct hfi1_devdata *dd,
5740 u32 context,
5741 const struct err_reg_info *eri)
5742{
5743 u64 reg;
5744 u32 count;
5745
5746 /* read in a loop until no more errors are seen */
5747 count = 0;
5748 while (1) {
5749 reg = read_kctxt_csr(dd, context, eri->status);
5750 if (reg == 0)
5751 break;
5752 write_kctxt_csr(dd, context, eri->clear, reg);
5753 if (likely(eri->handler))
5754 eri->handler(dd, context, reg);
5755 count++;
5756 if (count > MAX_CLEAR_COUNT) {
5757 u64 mask;
5758
5759 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
17fb4f29 5760 eri->desc, reg);
77241056
MM		 5761 /*
5762 * Read-modify-write so any other masked bits
5763 * remain masked.
5764 */
5765 mask = read_kctxt_csr(dd, context, eri->mask);
5766 mask &= ~reg;
5767 write_kctxt_csr(dd, context, eri->mask, mask);
5768 break;
5769 }
5770 }
5771}
5772
5773/*
5774 * CCE block "misc" interrupt. Source is < 16.
5775 */
5776static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5777{
5778 const struct err_reg_info *eri = &misc_errs[source];
5779
5780 if (eri->handler) {
5781 interrupt_clear_down(dd, 0, eri);
5782 } else {
5783 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
17fb4f29 5784 source);
77241056
MM		 5785 }
5786}
5787
5788static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5789{
5790 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5791 sc_err_status_flags,
5792 ARRAY_SIZE(sc_err_status_flags));
77241056
MM		 5793}
5794
5795/*
5796 * Send context error interrupt. Source (hw_context) is < 160.
5797 *
5798 * All send context errors cause the send context to halt. The normal
5799 * clear-down mechanism cannot be used because we cannot clear the
5800 * error bits until several other long-running items are done first.
5801 * This is OK because with the context halted, nothing else is going
5802 * to happen on it anyway.
5803 */
5804static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5805 unsigned int hw_context)
5806{
5807 struct send_context_info *sci;
5808 struct send_context *sc;
5809 char flags[96];
5810 u64 status;
5811 u32 sw_index;
2c5b521a 5812 int i = 0;
77241056
MM		 5813
5814 sw_index = dd->hw_to_sw[hw_context];
5815 if (sw_index >= dd->num_send_contexts) {
5816 dd_dev_err(dd,
17fb4f29
JJ		 5817 "out of range sw index %u for send context %u\n",
5818 sw_index, hw_context);
77241056
MM		 5819 return;
5820 }
5821 sci = &dd->send_contexts[sw_index];
5822 sc = sci->sc;
5823 if (!sc) {
5824 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
17fb4f29 5825 sw_index, hw_context);
77241056
MM		 5826 return;
5827 }
5828
5829 /* tell the software that a halt has begun */
5830 sc_stop(sc, SCF_HALTED);
5831
5832 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5833
5834 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
17fb4f29
JJ		 5835 send_context_err_status_string(flags, sizeof(flags),
5836 status));
77241056
MM		 5837
5838 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
69a00b8e 5839 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
77241056
MM		 5840
5841 /*
5842 * Automatically restart halted kernel contexts out of interrupt
5843 * context. User contexts must ask the driver to restart the context.
5844 */
5845 if (sc->type != SC_USER)
5846 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
2c5b521a
JR		 5847
5848 /*
5849 * Update the counters for the corresponding status bits.
5850 * Note that these particular counters are aggregated over all
5851 * 160 contexts.
5852 */
5853 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5854 if (status & (1ull << i))
5855 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
5856 }
77241056
MM		 5857}
5858
5859static void handle_sdma_eng_err(struct hfi1_devdata *dd,
5860 unsigned int source, u64 status)
5861{
5862 struct sdma_engine *sde;
2c5b521a 5863 int i = 0;
77241056
MM		 5864
5865 sde = &dd->per_sdma[source];
5866#ifdef CONFIG_SDMA_VERBOSITY
5867 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5868 slashstrip(__FILE__), __LINE__, __func__);
5869 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
5870 sde->this_idx, source, (unsigned long long)status);
5871#endif
a699c6c2 5872 sde->err_cnt++;
77241056 5873 sdma_engine_error(sde, status);
2c5b521a
JR		 5874
5875 /*
5876 * Update the counters for the corresponding status bits.
5877 * Note that these particular counters are aggregated over
5878 * all 16 DMA engines.
5879 */
5880 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
5881 if (status & (1ull << i))
5882 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
5883 }
77241056
MM		 5884}
5885
5886/*
5887 * CCE block SDMA error interrupt. Source is < 16.
5888 */
5889static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
5890{
5891#ifdef CONFIG_SDMA_VERBOSITY
5892 struct sdma_engine *sde = &dd->per_sdma[source];
5893
5894 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5895 slashstrip(__FILE__), __LINE__, __func__);
5896 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
5897 source);
5898 sdma_dumpstate(sde);
5899#endif
5900 interrupt_clear_down(dd, source, &sdma_eng_err);
5901}
5902
5903/*
5904 * CCE block "various" interrupt. Source is < 8.
5905 */
5906static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
5907{
5908 const struct err_reg_info *eri = &various_err[source];
5909
5910 /*
5911 * TCritInt cannot go through interrupt_clear_down()
5912 * because it is not a second tier interrupt. The handler
5913 * should be called directly.
5914 */
5915 if (source == TCRIT_INT_SOURCE)
5916 handle_temp_err(dd);
5917 else if (eri->handler)
5918 interrupt_clear_down(dd, 0, eri);
5919 else
5920 dd_dev_info(dd,
17fb4f29
JJ		 5921 "%s: Unimplemented/reserved interrupt %d\n",
5922 __func__, source);
77241056
MM		 5923}
5924
5925static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
5926{
8ebd4cf1 5927 /* src_ctx is always zero */
77241056
MM		 5928 struct hfi1_pportdata *ppd = dd->pport;
5929 unsigned long flags;
5930 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5931
5932 if (reg & QSFP_HFI0_MODPRST_N) {
77241056 5933 dd_dev_info(dd, "%s: ModPresent triggered QSFP interrupt\n",
17fb4f29 5934 __func__);
77241056
MM		 5935
5936 if (!qsfp_mod_present(ppd)) {
5937 ppd->driver_link_ready = 0;
5938 /*
5939 * Cable removed, reset all our information about the
5940 * cache and cable capabilities
5941 */
5942
5943 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5944 /*
5945 * We don't set cache_refresh_required here as we expect
5946 * an interrupt when a cable is inserted
5947 */
5948 ppd->qsfp_info.cache_valid = 0;
8ebd4cf1
EH		 5949 ppd->qsfp_info.reset_needed = 0;
5950 ppd->qsfp_info.limiting_active = 0;
77241056 5951 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 5952 flags);
8ebd4cf1
EH		 5953 /* Invert the ModPresent pin now to detect plug-in */
5954 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
5955 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
a9c05e35
BM		 5956
5957 if ((ppd->offline_disabled_reason >
5958 HFI1_ODR_MASK(
e1bf0d5e 5959 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
a9c05e35
BM		 5960 (ppd->offline_disabled_reason ==
5961 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
5962 ppd->offline_disabled_reason =
5963 HFI1_ODR_MASK(
e1bf0d5e 5964 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
a9c05e35 5965
77241056
MM		 5966 if (ppd->host_link_state == HLS_DN_POLL) {
5967 /*
5968 * The link is still in POLL. This means
5969 * that the normal link down processing
5970 * will not happen. We have to do it here
5971 * before turning the DC off.
5972 */
5973 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
5974 }
5975 } else {
5976 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5977 ppd->qsfp_info.cache_valid = 0;
5978 ppd->qsfp_info.cache_refresh_required = 1;
5979 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 5980 flags);
77241056 5981
8ebd4cf1
EH		 5982 /*
5983 * Stop inversion of ModPresent pin to detect
5984 * removal of the cable
5985 */
77241056 5986 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
8ebd4cf1
EH		 5987 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
5988 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
5989
5990 ppd->offline_disabled_reason =
5991 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
77241056
MM		 5992 }
5993 }
5994
5995 if (reg & QSFP_HFI0_INT_N) {
77241056 5996 dd_dev_info(dd, "%s: IntN triggered QSFP interrupt\n",
17fb4f29 5997 __func__);
77241056
MM		 5998 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5999 ppd->qsfp_info.check_interrupt_flags = 1;
77241056
MM		 6000 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6001 }
6002
6003 /* Schedule the QSFP work only if there is a cable attached. */
6004 if (qsfp_mod_present(ppd))
6005 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
6006}
6007
6008static int request_host_lcb_access(struct hfi1_devdata *dd)
6009{
6010 int ret;
6011
6012 ret = do_8051_command(dd, HCMD_MISC,
17fb4f29
JJ		 6013 (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6014 LOAD_DATA_FIELD_ID_SHIFT, NULL);
77241056
MM		 6015 if (ret != HCMD_SUCCESS) {
6016 dd_dev_err(dd, "%s: command failed with error %d\n",
17fb4f29 6017 __func__, ret);
77241056
MM		 6018 }
6019 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6020}
6021
6022static int request_8051_lcb_access(struct hfi1_devdata *dd)
6023{
6024 int ret;
6025
6026 ret = do_8051_command(dd, HCMD_MISC,
17fb4f29
JJ		 6027 (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6028 LOAD_DATA_FIELD_ID_SHIFT, NULL);
77241056
MM		 6029 if (ret != HCMD_SUCCESS) {
6030 dd_dev_err(dd, "%s: command failed with error %d\n",
17fb4f29 6031 __func__, ret);
77241056
MM		 6032 }
6033 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6034}
6035
6036/*
6037 * Set the LCB selector - allow host access. The DCC selector always
6038 * points to the host.
6039 */
6040static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6041{
6042 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
17fb4f29
JJ		 6043 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6044 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
77241056
MM		 6045}
6046
6047/*
6048 * Clear the LCB selector - allow 8051 access. The DCC selector always
6049 * points to the host.
6050 */
6051static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6052{
6053 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
17fb4f29 6054 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
77241056
MM		 6055}
6056
6057/*
6058 * Acquire LCB access from the 8051. If the host already has access,
6059 * just increment a counter. Otherwise, inform the 8051 that the
6060 * host is taking access.
6061 *
6062 * Returns:
6063 * 0 on success
6064 * -EBUSY if the 8051 has control and cannot be disturbed
6065 * -errno if unable to acquire access from the 8051
6066 */
6067int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6068{
6069 struct hfi1_pportdata *ppd = dd->pport;
6070 int ret = 0;
6071
6072 /*
6073 * Use the host link state lock so the operation of this routine
6074 * { link state check, selector change, count increment } can occur
6075 * as a unit against a link state change. Otherwise there is a
6076 * race between the state change and the count increment.
6077 */
6078 if (sleep_ok) {
6079 mutex_lock(&ppd->hls_lock);
6080 } else {
951842b0 6081 while (!mutex_trylock(&ppd->hls_lock))
77241056
MM		 6082 udelay(1);
6083 }
6084
6085 /* this access is valid only when the link is up */
6086 if ((ppd->host_link_state & HLS_UP) == 0) {
6087 dd_dev_info(dd, "%s: link state %s not up\n",
17fb4f29 6088 __func__, link_state_name(ppd->host_link_state));
77241056
MM		 6089 ret = -EBUSY;
6090 goto done;
6091 }
6092
6093 if (dd->lcb_access_count == 0) {
6094 ret = request_host_lcb_access(dd);
6095 if (ret) {
6096 dd_dev_err(dd,
17fb4f29
JJ		 6097 "%s: unable to acquire LCB access, err %d\n",
6098 __func__, ret);
77241056
MM		 6099 goto done;
6100 }
6101 set_host_lcb_access(dd);
6102 }
6103 dd->lcb_access_count++;
6104done:
6105 mutex_unlock(&ppd->hls_lock);
6106 return ret;
6107}
6108
6109/*
6110 * Release LCB access by decrementing the use count. If the count is moving
6111 * from 1 to 0, inform 8051 that it has control back.
6112 *
6113 * Returns:
6114 * 0 on success
6115 * -errno if unable to release access to the 8051
6116 */
6117int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6118{
6119 int ret = 0;
6120
6121 /*
6122 * Use the host link state lock because the acquire needed it.
6123 * Here, we only need to keep { selector change, count decrement }
6124 * as a unit.
6125 */
6126 if (sleep_ok) {
6127 mutex_lock(&dd->pport->hls_lock);
6128 } else {
951842b0 6129 while (!mutex_trylock(&dd->pport->hls_lock))
77241056
MM		 6130 udelay(1);
6131 }
6132
6133 if (dd->lcb_access_count == 0) {
6134 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
17fb4f29 6135 __func__);
77241056
MM		 6136 goto done;
6137 }
6138
6139 if (dd->lcb_access_count == 1) {
6140 set_8051_lcb_access(dd);
6141 ret = request_8051_lcb_access(dd);
6142 if (ret) {
6143 dd_dev_err(dd,
17fb4f29
JJ		 6144 "%s: unable to release LCB access, err %d\n",
6145 __func__, ret);
77241056
MM		 6146 /* restore host access if the grant didn't work */
6147 set_host_lcb_access(dd);
6148 goto done;
6149 }
6150 }
6151 dd->lcb_access_count--;
6152done:
6153 mutex_unlock(&dd->pport->hls_lock);
6154 return ret;
6155}
6156
6157/*
6158 * Initialize LCB access variables and state. Called during driver load,
6159 * after most of the initialization is finished.
6160 *
6161 * The DC default is LCB access on for the host. The driver defaults to
6162 * leaving access to the 8051. Assign access now - this constrains the call
6163 * to this routine to be after all LCB set-up is done. In particular, after
6164 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6165 */
6166static void init_lcb_access(struct hfi1_devdata *dd)
6167{
6168 dd->lcb_access_count = 0;
6169}
6170
6171/*
6172 * Write a response back to a 8051 request.
6173 */
6174static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6175{
6176 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
17fb4f29
JJ		 6177 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6178 (u64)return_code <<
6179 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6180 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
77241056
MM		 6181}
6182
6183/*
cbac386a
EH		 6184 * Handle host requests from the 8051.
6185 *
6186 * This is a work-queue function outside of the interrupt.
77241056 6187 */
cbac386a 6188void handle_8051_request(struct work_struct *work)
77241056 6189{
cbac386a
EH		 6190 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6191 dc_host_req_work);
6192 struct hfi1_devdata *dd = ppd->dd;
77241056 6193 u64 reg;
cbac386a
EH		 6194 u16 data = 0;
6195 u8 type, i, lanes, *cache = ppd->qsfp_info.cache;
6196 u8 cdr_ctrl_byte = cache[QSFP_CDR_CTRL_BYTE_OFFS];
77241056
MM		 6197
6198 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6199 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6200 return; /* no request */
6201
6202 /* zero out COMPLETED so the response is seen */
6203 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6204
6205 /* extract request details */
6206 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6207 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6208 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6209 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6210
6211 switch (type) {
6212 case HREQ_LOAD_CONFIG:
6213 case HREQ_SAVE_CONFIG:
6214 case HREQ_READ_CONFIG:
6215 case HREQ_SET_TX_EQ_ABS:
6216 case HREQ_SET_TX_EQ_REL:
77241056 6217 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
17fb4f29 6218 type);
77241056
MM		 6219 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6220 break;
6221
cbac386a
EH		 6222 case HREQ_ENABLE:
6223 lanes = data & 0xF;
6224 for (i = 0; lanes; lanes >>= 1, i++) {
6225 if (!(lanes & 1))
6226 continue;
6227 if (data & 0x200) {
6228 /* enable TX CDR */
6229 if (cache[QSFP_MOD_PWR_OFFS] & 0x8 &&
6230 cache[QSFP_CDR_INFO_OFFS] & 0x80)
6231 cdr_ctrl_byte |= (1 << (i + 4));
6232 } else {
6233 /* disable TX CDR */
6234 if (cache[QSFP_MOD_PWR_OFFS] & 0x8 &&
6235 cache[QSFP_CDR_INFO_OFFS] & 0x80)
6236 cdr_ctrl_byte &= ~(1 << (i + 4));
6237 }
6238
6239 if (data & 0x800) {
6240 /* enable RX CDR */
6241 if (cache[QSFP_MOD_PWR_OFFS] & 0x4 &&
6242 cache[QSFP_CDR_INFO_OFFS] & 0x40)
6243 cdr_ctrl_byte |= (1 << i);
6244 } else {
6245 /* disable RX CDR */
6246 if (cache[QSFP_MOD_PWR_OFFS] & 0x4 &&
6247 cache[QSFP_CDR_INFO_OFFS] & 0x40)
6248 cdr_ctrl_byte &= ~(1 << i);
6249 }
6250 }
6251 qsfp_write(ppd, ppd->dd->hfi1_id, QSFP_CDR_CTRL_BYTE_OFFS,
6252 &cdr_ctrl_byte, 1);
6253 hreq_response(dd, HREQ_SUCCESS, data);
6254 refresh_qsfp_cache(ppd, &ppd->qsfp_info);
6255 break;
6256
77241056
MM		 6257 case HREQ_CONFIG_DONE:
6258 hreq_response(dd, HREQ_SUCCESS, 0);
6259 break;
6260
6261 case HREQ_INTERFACE_TEST:
6262 hreq_response(dd, HREQ_SUCCESS, data);
6263 break;
6264
6265 default:
6266 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6267 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6268 break;
6269 }
6270}
6271
6272static void write_global_credit(struct hfi1_devdata *dd,
6273 u8 vau, u16 total, u16 shared)
6274{
6275 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
17fb4f29
JJ		 6276 ((u64)total <<
6277 SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) |
6278 ((u64)shared <<
6279 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) |
6280 ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
77241056
MM		 6281}
6282
6283/*
6284 * Set up initial VL15 credits of the remote. Assumes the rest of
6285 * the CM credit registers are zero from a previous global or credit reset .
6286 */
6287void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
6288{
6289 /* leave shared count at zero for both global and VL15 */
6290 write_global_credit(dd, vau, vl15buf, 0);
6291
6292 /* We may need some credits for another VL when sending packets
6293 * with the snoop interface. Dividing it down the middle for VL15
6294 * and VL0 should suffice.
6295 */
6296 if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) {
6297 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1)
6298 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6299 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1)
6300 << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT);
6301 } else {
6302 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6303 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6304 }
6305}
6306
6307/*
6308 * Zero all credit details from the previous connection and
6309 * reset the CM manager's internal counters.
6310 */
6311void reset_link_credits(struct hfi1_devdata *dd)
6312{
6313 int i;
6314
6315 /* remove all previous VL credit limits */
6316 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 6317 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
77241056
MM		 6318 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6319 write_global_credit(dd, 0, 0, 0);
6320 /* reset the CM block */
6321 pio_send_control(dd, PSC_CM_RESET);
6322}
6323
6324/* convert a vCU to a CU */
6325static u32 vcu_to_cu(u8 vcu)
6326{
6327 return 1 << vcu;
6328}
6329
6330/* convert a CU to a vCU */
6331static u8 cu_to_vcu(u32 cu)
6332{
6333 return ilog2(cu);
6334}
6335
6336/* convert a vAU to an AU */
6337static u32 vau_to_au(u8 vau)
6338{
6339 return 8 * (1 << vau);
6340}
6341
6342static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6343{
6344 ppd->sm_trap_qp = 0x0;
6345 ppd->sa_qp = 0x1;
6346}
6347
6348/*
6349 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
6350 */
6351static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6352{
6353 u64 reg;
6354
6355 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6356 write_csr(dd, DC_LCB_CFG_RUN, 0);
6357 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6358 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
17fb4f29 6359 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
77241056
MM		 6360 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6361 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6362 reg = read_csr(dd, DCC_CFG_RESET);
17fb4f29
JJ		 6363 write_csr(dd, DCC_CFG_RESET, reg |
6364 (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) |
6365 (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
50e5dcbe 6366 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
77241056
MM		 6367 if (!abort) {
6368 udelay(1); /* must hold for the longer of 16cclks or 20ns */
6369 write_csr(dd, DCC_CFG_RESET, reg);
6370 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6371 }
6372}
6373
6374/*
6375 * This routine should be called after the link has been transitioned to
6376 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6377 * reset).
6378 *
6379 * The expectation is that the caller of this routine would have taken
6380 * care of properly transitioning the link into the correct state.
6381 */
6382static void dc_shutdown(struct hfi1_devdata *dd)
6383{
6384 unsigned long flags;
6385
6386 spin_lock_irqsave(&dd->dc8051_lock, flags);
6387 if (dd->dc_shutdown) {
6388 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6389 return;
6390 }
6391 dd->dc_shutdown = 1;
6392 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6393 /* Shutdown the LCB */
6394 lcb_shutdown(dd, 1);
4d114fdd
JJ		 6395 /*
6396 * Going to OFFLINE would have causes the 8051 to put the
77241056 6397 * SerDes into reset already. Just need to shut down the 8051,
4d114fdd
JJ		 6398 * itself.
6399 */
77241056
MM		 6400 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6401}
6402
4d114fdd
JJ		 6403/*
6404 * Calling this after the DC has been brought out of reset should not
6405 * do any damage.
6406 */
77241056
MM		 6407static void dc_start(struct hfi1_devdata *dd)
6408{
6409 unsigned long flags;
6410 int ret;
6411
6412 spin_lock_irqsave(&dd->dc8051_lock, flags);
6413 if (!dd->dc_shutdown)
6414 goto done;
6415 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6416 /* Take the 8051 out of reset */
6417 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6418 /* Wait until 8051 is ready */
6419 ret = wait_fm_ready(dd, TIMEOUT_8051_START);
6420 if (ret) {
6421 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
17fb4f29 6422 __func__);
77241056
MM		 6423 }
6424 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6425 write_csr(dd, DCC_CFG_RESET, 0x10);
6426 /* lcb_shutdown() with abort=1 does not restore these */
6427 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6428 spin_lock_irqsave(&dd->dc8051_lock, flags);
6429 dd->dc_shutdown = 0;
6430done:
6431 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6432}
6433
6434/*
6435 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6436 */
6437static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6438{
6439 u64 rx_radr, tx_radr;
6440 u32 version;
6441
6442 if (dd->icode != ICODE_FPGA_EMULATION)
6443 return;
6444
6445 /*
6446 * These LCB defaults on emulator _s are good, nothing to do here:
6447 * LCB_CFG_TX_FIFOS_RADR
6448 * LCB_CFG_RX_FIFOS_RADR
6449 * LCB_CFG_LN_DCLK
6450 * LCB_CFG_IGNORE_LOST_RCLK
6451 */
6452 if (is_emulator_s(dd))
6453 return;
6454 /* else this is _p */
6455
6456 version = emulator_rev(dd);
995deafa 6457 if (!is_ax(dd))
77241056
MM		 6458 version = 0x2d; /* all B0 use 0x2d or higher settings */
6459
6460 if (version <= 0x12) {
6461 /* release 0x12 and below */
6462
6463 /*
6464 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6465 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6466 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6467 */
6468 rx_radr =
6469 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6470 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6471 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6472 /*
6473 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6474 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6475 */
6476 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6477 } else if (version <= 0x18) {
6478 /* release 0x13 up to 0x18 */
6479 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6480 rx_radr =
6481 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6482 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6483 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6484 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6485 } else if (version == 0x19) {
6486 /* release 0x19 */
6487 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6488 rx_radr =
6489 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6490 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6491 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6492 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6493 } else if (version == 0x1a) {
6494 /* release 0x1a */
6495 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6496 rx_radr =
6497 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6498 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6499 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6500 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6501 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6502 } else {
6503 /* release 0x1b and higher */
6504 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6505 rx_radr =
6506 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6507 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6508 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6509 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6510 }
6511
6512 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6513 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6514 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
17fb4f29 6515 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
77241056
MM		 6516 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6517}
6518
6519/*
6520 * Handle a SMA idle message
6521 *
6522 * This is a work-queue function outside of the interrupt.
6523 */
6524void handle_sma_message(struct work_struct *work)
6525{
6526 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6527 sma_message_work);
6528 struct hfi1_devdata *dd = ppd->dd;
6529 u64 msg;
6530 int ret;
6531
4d114fdd
JJ		 6532 /*
6533 * msg is bytes 1-4 of the 40-bit idle message - the command code
6534 * is stripped off
6535 */
77241056
MM		 6536 ret = read_idle_sma(dd, &msg);
6537 if (ret)
6538 return;
6539 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6540 /*
6541 * React to the SMA message. Byte[1] (0 for us) is the command.
6542 */
6543 switch (msg & 0xff) {
6544 case SMA_IDLE_ARM:
6545 /*
6546 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6547 * State Transitions
6548 *
6549 * Only expected in INIT or ARMED, discard otherwise.
6550 */
6551 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6552 ppd->neighbor_normal = 1;
6553 break;
6554 case SMA_IDLE_ACTIVE:
6555 /*
6556 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6557 * State Transitions
6558 *
6559 * Can activate the node. Discard otherwise.
6560 */
d0d236ea
JJ		 6561 if (ppd->host_link_state == HLS_UP_ARMED &&
6562 ppd->is_active_optimize_enabled) {
77241056
MM		 6563 ppd->neighbor_normal = 1;
6564 ret = set_link_state(ppd, HLS_UP_ACTIVE);
6565 if (ret)
6566 dd_dev_err(
6567 dd,
6568 "%s: received Active SMA idle message, couldn't set link to Active\n",
6569 __func__);
6570 }
6571 break;
6572 default:
6573 dd_dev_err(dd,
17fb4f29
JJ		 6574 "%s: received unexpected SMA idle message 0x%llx\n",
6575 __func__, msg);
77241056
MM		 6576 break;
6577 }
6578}
6579
6580static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6581{
6582 u64 rcvctrl;
6583 unsigned long flags;
6584
6585 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6586 rcvctrl = read_csr(dd, RCV_CTRL);
6587 rcvctrl |= add;
6588 rcvctrl &= ~clear;
6589 write_csr(dd, RCV_CTRL, rcvctrl);
6590 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6591}
6592
6593static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6594{
6595 adjust_rcvctrl(dd, add, 0);
6596}
6597
6598static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6599{
6600 adjust_rcvctrl(dd, 0, clear);
6601}
6602
6603/*
6604 * Called from all interrupt handlers to start handling an SPC freeze.
6605 */
6606void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6607{
6608 struct hfi1_devdata *dd = ppd->dd;
6609 struct send_context *sc;
6610 int i;
6611
6612 if (flags & FREEZE_SELF)
6613 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6614
6615 /* enter frozen mode */
6616 dd->flags |= HFI1_FROZEN;
6617
6618 /* notify all SDMA engines that they are going into a freeze */
6619 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6620
6621 /* do halt pre-handling on all enabled send contexts */
6622 for (i = 0; i < dd->num_send_contexts; i++) {
6623 sc = dd->send_contexts[i].sc;
6624 if (sc && (sc->flags & SCF_ENABLED))
6625 sc_stop(sc, SCF_FROZEN | SCF_HALTED);
6626 }
6627
6628 /* Send context are frozen. Notify user space */
6629 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6630
6631 if (flags & FREEZE_ABORT) {
6632 dd_dev_err(dd,
6633 "Aborted freeze recovery. Please REBOOT system\n");
6634 return;
6635 }
6636 /* queue non-interrupt handler */
6637 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6638}
6639
6640/*
6641 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6642 * depending on the "freeze" parameter.
6643 *
6644 * No need to return an error if it times out, our only option
6645 * is to proceed anyway.
6646 */
6647static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6648{
6649 unsigned long timeout;
6650 u64 reg;
6651
6652 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6653 while (1) {
6654 reg = read_csr(dd, CCE_STATUS);
6655 if (freeze) {
6656 /* waiting until all indicators are set */
6657 if ((reg & ALL_FROZE) == ALL_FROZE)
6658 return; /* all done */
6659 } else {
6660 /* waiting until all indicators are clear */
6661 if ((reg & ALL_FROZE) == 0)
6662 return; /* all done */
6663 }
6664
6665 if (time_after(jiffies, timeout)) {
6666 dd_dev_err(dd,
17fb4f29
JJ		 6667 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6668 freeze ? "" : "un", reg & ALL_FROZE,
6669 freeze ? ALL_FROZE : 0ull);
77241056
MM		 6670 return;
6671 }
6672 usleep_range(80, 120);
6673 }
6674}
6675
6676/*
6677 * Do all freeze handling for the RXE block.
6678 */
6679static void rxe_freeze(struct hfi1_devdata *dd)
6680{
6681 int i;
6682
6683 /* disable port */
6684 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6685
6686 /* disable all receive contexts */
6687 for (i = 0; i < dd->num_rcv_contexts; i++)
6688 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
6689}
6690
6691/*
6692 * Unfreeze handling for the RXE block - kernel contexts only.
6693 * This will also enable the port. User contexts will do unfreeze
6694 * handling on a per-context basis as they call into the driver.
6695 *
6696 */
6697static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6698{
566c157c 6699 u32 rcvmask;
77241056
MM		 6700 int i;
6701
6702 /* enable all kernel contexts */
566c157c
MH		 6703 for (i = 0; i < dd->n_krcv_queues; i++) {
6704 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6705 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6706 rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ?
6707 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6708 hfi1_rcvctrl(dd, rcvmask, i);
6709 }
77241056
MM		 6710
6711 /* enable port */
6712 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6713}
6714
6715/*
6716 * Non-interrupt SPC freeze handling.
6717 *
6718 * This is a work-queue function outside of the triggering interrupt.
6719 */
6720void handle_freeze(struct work_struct *work)
6721{
6722 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6723 freeze_work);
6724 struct hfi1_devdata *dd = ppd->dd;
6725
6726 /* wait for freeze indicators on all affected blocks */
77241056
MM		 6727 wait_for_freeze_status(dd, 1);
6728
6729 /* SPC is now frozen */
6730
6731 /* do send PIO freeze steps */
6732 pio_freeze(dd);
6733
6734 /* do send DMA freeze steps */
6735 sdma_freeze(dd);
6736
6737 /* do send egress freeze steps - nothing to do */
6738
6739 /* do receive freeze steps */
6740 rxe_freeze(dd);
6741
6742 /*
6743 * Unfreeze the hardware - clear the freeze, wait for each
6744 * block's frozen bit to clear, then clear the frozen flag.
6745 */
6746 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6747 wait_for_freeze_status(dd, 0);
6748
995deafa 6749 if (is_ax(dd)) {
77241056
MM		 6750 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6751 wait_for_freeze_status(dd, 1);
6752 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6753 wait_for_freeze_status(dd, 0);
6754 }
6755
6756 /* do send PIO unfreeze steps for kernel contexts */
6757 pio_kernel_unfreeze(dd);
6758
6759 /* do send DMA unfreeze steps */
6760 sdma_unfreeze(dd);
6761
6762 /* do send egress unfreeze steps - nothing to do */
6763
6764 /* do receive unfreeze steps for kernel contexts */
6765 rxe_kernel_unfreeze(dd);
6766
6767 /*
6768 * The unfreeze procedure touches global device registers when
6769 * it disables and re-enables RXE. Mark the device unfrozen
6770 * after all that is done so other parts of the driver waiting
6771 * for the device to unfreeze don't do things out of order.
6772 *
6773 * The above implies that the meaning of HFI1_FROZEN flag is
6774 * "Device has gone into freeze mode and freeze mode handling
6775 * is still in progress."
6776 *
6777 * The flag will be removed when freeze mode processing has
6778 * completed.
6779 */
6780 dd->flags &= ~HFI1_FROZEN;
6781 wake_up(&dd->event_queue);
6782
6783 /* no longer frozen */
77241056
MM		 6784}
6785
6786/*
6787 * Handle a link up interrupt from the 8051.
6788 *
6789 * This is a work-queue function outside of the interrupt.
6790 */
6791void handle_link_up(struct work_struct *work)
6792{
6793 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
17fb4f29 6794 link_up_work);
77241056
MM		 6795 set_link_state(ppd, HLS_UP_INIT);
6796
6797 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6798 read_ltp_rtt(ppd->dd);
6799 /*
6800 * OPA specifies that certain counters are cleared on a transition
6801 * to link up, so do that.
6802 */
6803 clear_linkup_counters(ppd->dd);
6804 /*
6805 * And (re)set link up default values.
6806 */
6807 set_linkup_defaults(ppd);
6808
6809 /* enforce link speed enabled */
6810 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6811 /* oops - current speed is not enabled, bounce */
6812 dd_dev_err(ppd->dd,
17fb4f29
JJ		 6813 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6814 ppd->link_speed_active, ppd->link_speed_enabled);
77241056 6815 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
17fb4f29 6816 OPA_LINKDOWN_REASON_SPEED_POLICY);
77241056 6817 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 6818 tune_serdes(ppd);
77241056
MM		 6819 start_link(ppd);
6820 }
6821}
6822
4d114fdd
JJ		 6823/*
6824 * Several pieces of LNI information were cached for SMA in ppd.
6825 * Reset these on link down
6826 */
77241056
MM		 6827static void reset_neighbor_info(struct hfi1_pportdata *ppd)
6828{
6829 ppd->neighbor_guid = 0;
6830 ppd->neighbor_port_number = 0;
6831 ppd->neighbor_type = 0;
6832 ppd->neighbor_fm_security = 0;
6833}
6834
6835/*
6836 * Handle a link down interrupt from the 8051.
6837 *
6838 * This is a work-queue function outside of the interrupt.
6839 */
6840void handle_link_down(struct work_struct *work)
6841{
6842 u8 lcl_reason, neigh_reason = 0;
6843 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6844 link_down_work);
6845
8ebd4cf1
EH		 6846 if ((ppd->host_link_state &
6847 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
6848 ppd->port_type == PORT_TYPE_FIXED)
6849 ppd->offline_disabled_reason =
6850 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
6851
6852 /* Go offline first, then deal with reading/writing through 8051 */
77241056
MM		 6853 set_link_state(ppd, HLS_DN_OFFLINE);
6854
6855 lcl_reason = 0;
6856 read_planned_down_reason_code(ppd->dd, &neigh_reason);
6857
6858 /*
6859 * If no reason, assume peer-initiated but missed
6860 * LinkGoingDown idle flits.
6861 */
6862 if (neigh_reason == 0)
6863 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
6864
6865 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
6866
6867 reset_neighbor_info(ppd);
6868
6869 /* disable the port */
6870 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6871
4d114fdd
JJ		 6872 /*
6873 * If there is no cable attached, turn the DC off. Otherwise,
6874 * start the link bring up.
6875 */
8ebd4cf1 6876 if (!qsfp_mod_present(ppd)) {
77241056 6877 dc_shutdown(ppd->dd);
8ebd4cf1
EH		 6878 } else {
6879 tune_serdes(ppd);
77241056 6880 start_link(ppd);
8ebd4cf1 6881 }
77241056
MM		 6882}
6883
6884void handle_link_bounce(struct work_struct *work)
6885{
6886 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6887 link_bounce_work);
6888
6889 /*
6890 * Only do something if the link is currently up.
6891 */
6892 if (ppd->host_link_state & HLS_UP) {
6893 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 6894 tune_serdes(ppd);
77241056
MM		 6895 start_link(ppd);
6896 } else {
6897 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
17fb4f29 6898 __func__, link_state_name(ppd->host_link_state));
77241056
MM		 6899 }
6900}
6901
6902/*
6903 * Mask conversion: Capability exchange to Port LTP. The capability
6904 * exchange has an implicit 16b CRC that is mandatory.
6905 */
6906static int cap_to_port_ltp(int cap)
6907{
6908 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
6909
6910 if (cap & CAP_CRC_14B)
6911 port_ltp |= PORT_LTP_CRC_MODE_14;
6912 if (cap & CAP_CRC_48B)
6913 port_ltp |= PORT_LTP_CRC_MODE_48;
6914 if (cap & CAP_CRC_12B_16B_PER_LANE)
6915 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
6916
6917 return port_ltp;
6918}
6919
6920/*
6921 * Convert an OPA Port LTP mask to capability mask
6922 */
6923int port_ltp_to_cap(int port_ltp)
6924{
6925 int cap_mask = 0;
6926
6927 if (port_ltp & PORT_LTP_CRC_MODE_14)
6928 cap_mask |= CAP_CRC_14B;
6929 if (port_ltp & PORT_LTP_CRC_MODE_48)
6930 cap_mask |= CAP_CRC_48B;
6931 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
6932 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
6933
6934 return cap_mask;
6935}
6936
6937/*
6938 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
6939 */
6940static int lcb_to_port_ltp(int lcb_crc)
6941{
6942 int port_ltp = 0;
6943
6944 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
6945 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
6946 else if (lcb_crc == LCB_CRC_48B)
6947 port_ltp = PORT_LTP_CRC_MODE_48;
6948 else if (lcb_crc == LCB_CRC_14B)
6949 port_ltp = PORT_LTP_CRC_MODE_14;
6950 else
6951 port_ltp = PORT_LTP_CRC_MODE_16;
6952
6953 return port_ltp;
6954}
6955
6956/*
6957 * Our neighbor has indicated that we are allowed to act as a fabric
6958 * manager, so place the full management partition key in the second
6959 * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
6960 * that we should already have the limited management partition key in
6961 * array element 1, and also that the port is not yet up when
6962 * add_full_mgmt_pkey() is invoked.
6963 */
6964static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
6965{
6966 struct hfi1_devdata *dd = ppd->dd;
6967
8764522e
DL		 6968 /* Sanity check - ppd->pkeys[2] should be 0, or already initalized */
6969 if (!((ppd->pkeys[2] == 0) || (ppd->pkeys[2] == FULL_MGMT_P_KEY)))
6970 dd_dev_warn(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
6971 __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
77241056
MM		 6972 ppd->pkeys[2] = FULL_MGMT_P_KEY;
6973 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
6974}
6975
6976/*
6977 * Convert the given link width to the OPA link width bitmask.
6978 */
6979static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
6980{
6981 switch (width) {
6982 case 0:
6983 /*
6984 * Simulator and quick linkup do not set the width.
6985 * Just set it to 4x without complaint.
6986 */
6987 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
6988 return OPA_LINK_WIDTH_4X;
6989 return 0; /* no lanes up */
6990 case 1: return OPA_LINK_WIDTH_1X;
6991 case 2: return OPA_LINK_WIDTH_2X;
6992 case 3: return OPA_LINK_WIDTH_3X;
6993 default:
6994 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
17fb4f29 6995 __func__, width);
77241056
MM		 6996 /* fall through */
6997 case 4: return OPA_LINK_WIDTH_4X;
6998 }
6999}
7000
7001/*
7002 * Do a population count on the bottom nibble.
7003 */
7004static const u8 bit_counts[16] = {
7005 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7006};
f4d507cd 7007
77241056
MM		 7008static inline u8 nibble_to_count(u8 nibble)
7009{
7010 return bit_counts[nibble & 0xf];
7011}
7012
7013/*
7014 * Read the active lane information from the 8051 registers and return
7015 * their widths.
7016 *
7017 * Active lane information is found in these 8051 registers:
7018 * enable_lane_tx
7019 * enable_lane_rx
7020 */
7021static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7022 u16 *rx_width)
7023{
7024 u16 tx, rx;
7025 u8 enable_lane_rx;
7026 u8 enable_lane_tx;
7027 u8 tx_polarity_inversion;
7028 u8 rx_polarity_inversion;
7029 u8 max_rate;
7030
7031 /* read the active lanes */
7032 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
17fb4f29 7033 &rx_polarity_inversion, &max_rate);
77241056
MM		 7034 read_local_lni(dd, &enable_lane_rx);
7035
7036 /* convert to counts */
7037 tx = nibble_to_count(enable_lane_tx);
7038 rx = nibble_to_count(enable_lane_rx);
7039
7040 /*
7041 * Set link_speed_active here, overriding what was set in
7042 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
7043 * set the max_rate field in handle_verify_cap until v0.19.
7044 */
d0d236ea
JJ		 7045 if ((dd->icode == ICODE_RTL_SILICON) &&
7046 (dd->dc8051_ver < dc8051_ver(0, 19))) {
77241056
MM		 7047 /* max_rate: 0 = 12.5G, 1 = 25G */
7048 switch (max_rate) {
7049 case 0:
7050 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7051 break;
7052 default:
7053 dd_dev_err(dd,
17fb4f29
JJ		 7054 "%s: unexpected max rate %d, using 25Gb\n",
7055 __func__, (int)max_rate);
77241056
MM		 7056 /* fall through */
7057 case 1:
7058 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7059 break;
7060 }
7061 }
7062
7063 dd_dev_info(dd,
17fb4f29
JJ		 7064 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7065 enable_lane_tx, tx, enable_lane_rx, rx);
77241056
MM		 7066 *tx_width = link_width_to_bits(dd, tx);
7067 *rx_width = link_width_to_bits(dd, rx);
7068}
7069
7070/*
7071 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7072 * Valid after the end of VerifyCap and during LinkUp. Does not change
7073 * after link up. I.e. look elsewhere for downgrade information.
7074 *
7075 * Bits are:
7076 * + bits [7:4] contain the number of active transmitters
7077 * + bits [3:0] contain the number of active receivers
7078 * These are numbers 1 through 4 and can be different values if the
7079 * link is asymmetric.
7080 *
7081 * verify_cap_local_fm_link_width[0] retains its original value.
7082 */
7083static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7084 u16 *rx_width)
7085{
7086 u16 widths, tx, rx;
7087 u8 misc_bits, local_flags;
7088 u16 active_tx, active_rx;
7089
7090 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
7091 tx = widths >> 12;
7092 rx = (widths >> 8) & 0xf;
7093
7094 *tx_width = link_width_to_bits(dd, tx);
7095 *rx_width = link_width_to_bits(dd, rx);
7096
7097 /* print the active widths */
7098 get_link_widths(dd, &active_tx, &active_rx);
7099}
7100
7101/*
7102 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7103 * hardware information when the link first comes up.
7104 *
7105 * The link width is not available until after VerifyCap.AllFramesReceived
7106 * (the trigger for handle_verify_cap), so this is outside that routine
7107 * and should be called when the 8051 signals linkup.
7108 */
7109void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7110{
7111 u16 tx_width, rx_width;
7112
7113 /* get end-of-LNI link widths */
7114 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7115
7116 /* use tx_width as the link is supposed to be symmetric on link up */
7117 ppd->link_width_active = tx_width;
7118 /* link width downgrade active (LWD.A) starts out matching LW.A */
7119 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7120 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7121 /* per OPA spec, on link up LWD.E resets to LWD.S */
7122 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7123 /* cache the active egress rate (units {10^6 bits/sec]) */
7124 ppd->current_egress_rate = active_egress_rate(ppd);
7125}
7126
7127/*
7128 * Handle a verify capabilities interrupt from the 8051.
7129 *
7130 * This is a work-queue function outside of the interrupt.
7131 */
7132void handle_verify_cap(struct work_struct *work)
7133{
7134 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7135 link_vc_work);
7136 struct hfi1_devdata *dd = ppd->dd;
7137 u64 reg;
7138 u8 power_management;
7139 u8 continious;
7140 u8 vcu;
7141 u8 vau;
7142 u8 z;
7143 u16 vl15buf;
7144 u16 link_widths;
7145 u16 crc_mask;
7146 u16 crc_val;
7147 u16 device_id;
7148 u16 active_tx, active_rx;
7149 u8 partner_supported_crc;
7150 u8 remote_tx_rate;
7151 u8 device_rev;
7152
7153 set_link_state(ppd, HLS_VERIFY_CAP);
7154
7155 lcb_shutdown(dd, 0);
7156 adjust_lcb_for_fpga_serdes(dd);
7157
7158 /*
7159 * These are now valid:
7160 * remote VerifyCap fields in the general LNI config
7161 * CSR DC8051_STS_REMOTE_GUID
7162 * CSR DC8051_STS_REMOTE_NODE_TYPE
7163 * CSR DC8051_STS_REMOTE_FM_SECURITY
7164 * CSR DC8051_STS_REMOTE_PORT_NO
7165 */
7166
7167 read_vc_remote_phy(dd, &power_management, &continious);
17fb4f29
JJ		 7168 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7169 &partner_supported_crc);
77241056
MM		 7170 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7171 read_remote_device_id(dd, &device_id, &device_rev);
7172 /*
7173 * And the 'MgmtAllowed' information, which is exchanged during
7174 * LNI, is also be available at this point.
7175 */
7176 read_mgmt_allowed(dd, &ppd->mgmt_allowed);
7177 /* print the active widths */
7178 get_link_widths(dd, &active_tx, &active_rx);
7179 dd_dev_info(dd,
17fb4f29
JJ		 7180 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7181 (int)power_management, (int)continious);
77241056 7182 dd_dev_info(dd,
17fb4f29
JJ		 7183 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7184 (int)vau, (int)z, (int)vcu, (int)vl15buf,
7185 (int)partner_supported_crc);
77241056 7186 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
17fb4f29 7187 (u32)remote_tx_rate, (u32)link_widths);
77241056 7188 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
17fb4f29 7189 (u32)device_id, (u32)device_rev);
77241056
MM		 7190 /*
7191 * The peer vAU value just read is the peer receiver value. HFI does
7192 * not support a transmit vAU of 0 (AU == 8). We advertised that
7193 * with Z=1 in the fabric capabilities sent to the peer. The peer
7194 * will see our Z=1, and, if it advertised a vAU of 0, will move its
7195 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
7196 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7197 * subject to the Z value exception.
7198 */
7199 if (vau == 0)
7200 vau = 1;
7201 set_up_vl15(dd, vau, vl15buf);
7202
7203 /* set up the LCB CRC mode */
7204 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7205
7206 /* order is important: use the lowest bit in common */
7207 if (crc_mask & CAP_CRC_14B)
7208 crc_val = LCB_CRC_14B;
7209 else if (crc_mask & CAP_CRC_48B)
7210 crc_val = LCB_CRC_48B;
7211 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7212 crc_val = LCB_CRC_12B_16B_PER_LANE;
7213 else
7214 crc_val = LCB_CRC_16B;
7215
7216 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7217 write_csr(dd, DC_LCB_CFG_CRC_MODE,
7218 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7219
7220 /* set (14b only) or clear sideband credit */
7221 reg = read_csr(dd, SEND_CM_CTRL);
7222 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7223 write_csr(dd, SEND_CM_CTRL,
17fb4f29 7224 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
77241056
MM		 7225 } else {
7226 write_csr(dd, SEND_CM_CTRL,
17fb4f29 7227 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
77241056
MM		 7228 }
7229
7230 ppd->link_speed_active = 0; /* invalid value */
7231 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
7232 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7233 switch (remote_tx_rate) {
7234 case 0:
7235 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7236 break;
7237 case 1:
7238 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7239 break;
7240 }
7241 } else {
7242 /* actual rate is highest bit of the ANDed rates */
7243 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7244
7245 if (rate & 2)
7246 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7247 else if (rate & 1)
7248 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7249 }
7250 if (ppd->link_speed_active == 0) {
7251 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
17fb4f29 7252 __func__, (int)remote_tx_rate);
77241056
MM		 7253 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7254 }
7255
7256 /*
7257 * Cache the values of the supported, enabled, and active
7258 * LTP CRC modes to return in 'portinfo' queries. But the bit
7259 * flags that are returned in the portinfo query differ from
7260 * what's in the link_crc_mask, crc_sizes, and crc_val
7261 * variables. Convert these here.
7262 */
7263 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7264 /* supported crc modes */
7265 ppd->port_ltp_crc_mode |=
7266 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7267 /* enabled crc modes */
7268 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7269 /* active crc mode */
7270
7271 /* set up the remote credit return table */
7272 assign_remote_cm_au_table(dd, vcu);
7273
7274 /*
7275 * The LCB is reset on entry to handle_verify_cap(), so this must
7276 * be applied on every link up.
7277 *
7278 * Adjust LCB error kill enable to kill the link if
7279 * these RBUF errors are seen:
7280 * REPLAY_BUF_MBE_SMASK
7281 * FLIT_INPUT_BUF_MBE_SMASK
7282 */
995deafa 7283 if (is_ax(dd)) { /* fixed in B0 */
77241056
MM		 7284 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7285 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7286 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7287 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7288 }
7289
7290 /* pull LCB fifos out of reset - all fifo clocks must be stable */
7291 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7292
7293 /* give 8051 access to the LCB CSRs */
7294 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7295 set_8051_lcb_access(dd);
7296
7297 ppd->neighbor_guid =
7298 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
7299 ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
7300 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
7301 ppd->neighbor_type =
7302 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
7303 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
7304 ppd->neighbor_fm_security =
7305 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
7306 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
7307 dd_dev_info(dd,
17fb4f29
JJ		 7308 "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
7309 ppd->neighbor_guid, ppd->neighbor_type,
7310 ppd->mgmt_allowed, ppd->neighbor_fm_security);
77241056
MM		 7311 if (ppd->mgmt_allowed)
7312 add_full_mgmt_pkey(ppd);
7313
7314 /* tell the 8051 to go to LinkUp */
7315 set_link_state(ppd, HLS_GOING_UP);
7316}
7317
7318/*
7319 * Apply the link width downgrade enabled policy against the current active
7320 * link widths.
7321 *
7322 * Called when the enabled policy changes or the active link widths change.
7323 */
7324void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
7325{
77241056 7326 int do_bounce = 0;
323fd785
DL		 7327 int tries;
7328 u16 lwde;
77241056
MM		 7329 u16 tx, rx;
7330
323fd785
DL		 7331 /* use the hls lock to avoid a race with actual link up */
7332 tries = 0;
7333retry:
77241056
MM		 7334 mutex_lock(&ppd->hls_lock);
7335 /* only apply if the link is up */
323fd785
DL		 7336 if (!(ppd->host_link_state & HLS_UP)) {
7337 /* still going up..wait and retry */
7338 if (ppd->host_link_state & HLS_GOING_UP) {
7339 if (++tries < 1000) {
7340 mutex_unlock(&ppd->hls_lock);
7341 usleep_range(100, 120); /* arbitrary */
7342 goto retry;
7343 }
7344 dd_dev_err(ppd->dd,
7345 "%s: giving up waiting for link state change\n",
7346 __func__);
7347 }
7348 goto done;
7349 }
7350
7351 lwde = ppd->link_width_downgrade_enabled;
77241056
MM		 7352
7353 if (refresh_widths) {
7354 get_link_widths(ppd->dd, &tx, &rx);
7355 ppd->link_width_downgrade_tx_active = tx;
7356 ppd->link_width_downgrade_rx_active = rx;
7357 }
7358
7359 if (lwde == 0) {
7360 /* downgrade is disabled */
7361
7362 /* bounce if not at starting active width */
17fb4f29
JJ		 7363 if ((ppd->link_width_active !=
7364 ppd->link_width_downgrade_tx_active) ||
7365 (ppd->link_width_active !=
7366 ppd->link_width_downgrade_rx_active)) {
77241056 7367 dd_dev_err(ppd->dd,
17fb4f29 7368 "Link downgrade is disabled and link has downgraded, downing link\n");
77241056 7369 dd_dev_err(ppd->dd,
17fb4f29
JJ		 7370 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
7371 ppd->link_width_active,
7372 ppd->link_width_downgrade_tx_active,
7373 ppd->link_width_downgrade_rx_active);
77241056
MM		 7374 do_bounce = 1;
7375 }
d0d236ea
JJ		 7376 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7377 (lwde & ppd->link_width_downgrade_rx_active) == 0) {
77241056
MM		 7378 /* Tx or Rx is outside the enabled policy */
7379 dd_dev_err(ppd->dd,
17fb4f29 7380 "Link is outside of downgrade allowed, downing link\n");
77241056 7381 dd_dev_err(ppd->dd,
17fb4f29
JJ		 7382 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7383 lwde, ppd->link_width_downgrade_tx_active,
7384 ppd->link_width_downgrade_rx_active);
77241056
MM		 7385 do_bounce = 1;
7386 }
7387
323fd785
DL		 7388done:
7389 mutex_unlock(&ppd->hls_lock);
7390
77241056
MM		 7391 if (do_bounce) {
7392 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
17fb4f29 7393 OPA_LINKDOWN_REASON_WIDTH_POLICY);
77241056 7394 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 7395 tune_serdes(ppd);
77241056
MM		 7396 start_link(ppd);
7397 }
7398}
7399
7400/*
7401 * Handle a link downgrade interrupt from the 8051.
7402 *
7403 * This is a work-queue function outside of the interrupt.
7404 */
7405void handle_link_downgrade(struct work_struct *work)
7406{
7407 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7408 link_downgrade_work);
7409
7410 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7411 apply_link_downgrade_policy(ppd, 1);
7412}
7413
7414static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7415{
7416 return flag_string(buf, buf_len, flags, dcc_err_flags,
7417 ARRAY_SIZE(dcc_err_flags));
7418}
7419
7420static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7421{
7422 return flag_string(buf, buf_len, flags, lcb_err_flags,
7423 ARRAY_SIZE(lcb_err_flags));
7424}
7425
7426static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7427{
7428 return flag_string(buf, buf_len, flags, dc8051_err_flags,
7429 ARRAY_SIZE(dc8051_err_flags));
7430}
7431
7432static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7433{
7434 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7435 ARRAY_SIZE(dc8051_info_err_flags));
7436}
7437
7438static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7439{
7440 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7441 ARRAY_SIZE(dc8051_info_host_msg_flags));
7442}
7443
7444static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7445{
7446 struct hfi1_pportdata *ppd = dd->pport;
7447 u64 info, err, host_msg;
7448 int queue_link_down = 0;
7449 char buf[96];
7450
7451 /* look at the flags */
7452 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7453 /* 8051 information set by firmware */
7454 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7455 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7456 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7457 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7458 host_msg = (info >>
7459 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7460 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7461
7462 /*
7463 * Handle error flags.
7464 */
7465 if (err & FAILED_LNI) {
7466 /*
7467 * LNI error indications are cleared by the 8051
7468 * only when starting polling. Only pay attention
7469 * to them when in the states that occur during
7470 * LNI.
7471 */
7472 if (ppd->host_link_state
7473 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7474 queue_link_down = 1;
7475 dd_dev_info(dd, "Link error: %s\n",
17fb4f29
JJ		 7476 dc8051_info_err_string(buf,
7477 sizeof(buf),
7478 err &
7479 FAILED_LNI));
77241056
MM		 7480 }
7481 err &= ~(u64)FAILED_LNI;
7482 }
6d014530
DL		 7483 /* unknown frames can happen durning LNI, just count */
7484 if (err & UNKNOWN_FRAME) {
7485 ppd->unknown_frame_count++;
7486 err &= ~(u64)UNKNOWN_FRAME;
7487 }
77241056
MM		 7488 if (err) {
7489 /* report remaining errors, but do not do anything */
7490 dd_dev_err(dd, "8051 info error: %s\n",
17fb4f29
JJ		 7491 dc8051_info_err_string(buf, sizeof(buf),
7492 err));
77241056
MM		 7493 }
7494
7495 /*
7496 * Handle host message flags.
7497 */
7498 if (host_msg & HOST_REQ_DONE) {
7499 /*
7500 * Presently, the driver does a busy wait for
7501 * host requests to complete. This is only an
7502 * informational message.
7503 * NOTE: The 8051 clears the host message
7504 * information *on the next 8051 command*.
7505 * Therefore, when linkup is achieved,
7506 * this flag will still be set.
7507 */
7508 host_msg &= ~(u64)HOST_REQ_DONE;
7509 }
7510 if (host_msg & BC_SMA_MSG) {
7511 queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
7512 host_msg &= ~(u64)BC_SMA_MSG;
7513 }
7514 if (host_msg & LINKUP_ACHIEVED) {
7515 dd_dev_info(dd, "8051: Link up\n");
7516 queue_work(ppd->hfi1_wq, &ppd->link_up_work);
7517 host_msg &= ~(u64)LINKUP_ACHIEVED;
7518 }
7519 if (host_msg & EXT_DEVICE_CFG_REQ) {
cbac386a 7520 queue_work(ppd->hfi1_wq, &ppd->dc_host_req_work);
77241056
MM		 7521 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7522 }
7523 if (host_msg & VERIFY_CAP_FRAME) {
7524 queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
7525 host_msg &= ~(u64)VERIFY_CAP_FRAME;
7526 }
7527 if (host_msg & LINK_GOING_DOWN) {
7528 const char *extra = "";
7529 /* no downgrade action needed if going down */
7530 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7531 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7532 extra = " (ignoring downgrade)";
7533 }
7534 dd_dev_info(dd, "8051: Link down%s\n", extra);
7535 queue_link_down = 1;
7536 host_msg &= ~(u64)LINK_GOING_DOWN;
7537 }
7538 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7539 queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
7540 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7541 }
7542 if (host_msg) {
7543 /* report remaining messages, but do not do anything */
7544 dd_dev_info(dd, "8051 info host message: %s\n",
17fb4f29
JJ		 7545 dc8051_info_host_msg_string(buf,
7546 sizeof(buf),
7547 host_msg));
77241056
MM		 7548 }
7549
7550 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7551 }
7552 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7553 /*
7554 * Lost the 8051 heartbeat. If this happens, we
7555 * receive constant interrupts about it. Disable
7556 * the interrupt after the first.
7557 */
7558 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7559 write_csr(dd, DC_DC8051_ERR_EN,
17fb4f29
JJ		 7560 read_csr(dd, DC_DC8051_ERR_EN) &
7561 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
77241056
MM		 7562
7563 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7564 }
7565 if (reg) {
7566 /* report the error, but do not do anything */
7567 dd_dev_err(dd, "8051 error: %s\n",
17fb4f29 7568 dc8051_err_string(buf, sizeof(buf), reg));
77241056
MM		 7569 }
7570
7571 if (queue_link_down) {
4d114fdd
JJ		 7572 /*
7573 * if the link is already going down or disabled, do not
7574 * queue another
7575 */
d0d236ea
JJ		 7576 if ((ppd->host_link_state &
7577 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7578 ppd->link_enabled == 0) {
77241056 7579 dd_dev_info(dd, "%s: not queuing link down\n",
17fb4f29 7580 __func__);
77241056
MM		 7581 } else {
7582 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
7583 }
7584 }
7585}
7586
7587static const char * const fm_config_txt[] = {
7588[0] =
7589 "BadHeadDist: Distance violation between two head flits",
7590[1] =
7591 "BadTailDist: Distance violation between two tail flits",
7592[2] =
7593 "BadCtrlDist: Distance violation between two credit control flits",
7594[3] =
7595 "BadCrdAck: Credits return for unsupported VL",
7596[4] =
7597 "UnsupportedVLMarker: Received VL Marker",
7598[5] =
7599 "BadPreempt: Exceeded the preemption nesting level",
7600[6] =
7601 "BadControlFlit: Received unsupported control flit",
7602/* no 7 */
7603[8] =
7604 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7605};
7606
7607static const char * const port_rcv_txt[] = {
7608[1] =
7609 "BadPktLen: Illegal PktLen",
7610[2] =
7611 "PktLenTooLong: Packet longer than PktLen",
7612[3] =
7613 "PktLenTooShort: Packet shorter than PktLen",
7614[4] =
7615 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7616[5] =
7617 "BadDLID: Illegal DLID (0, doesn't match HFI)",
7618[6] =
7619 "BadL2: Illegal L2 opcode",
7620[7] =
7621 "BadSC: Unsupported SC",
7622[9] =
7623 "BadRC: Illegal RC",
7624[11] =
7625 "PreemptError: Preempting with same VL",
7626[12] =
7627 "PreemptVL15: Preempting a VL15 packet",
7628};
7629
7630#define OPA_LDR_FMCONFIG_OFFSET 16
7631#define OPA_LDR_PORTRCV_OFFSET 0
7632static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7633{
7634 u64 info, hdr0, hdr1;
7635 const char *extra;
7636 char buf[96];
7637 struct hfi1_pportdata *ppd = dd->pport;
7638 u8 lcl_reason = 0;
7639 int do_bounce = 0;
7640
7641 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7642 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7643 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7644 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7645 /* set status bit */
7646 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7647 }
7648 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7649 }
7650
7651 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7652 struct hfi1_pportdata *ppd = dd->pport;
7653 /* this counter saturates at (2^32) - 1 */
7654 if (ppd->link_downed < (u32)UINT_MAX)
7655 ppd->link_downed++;
7656 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7657 }
7658
7659 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7660 u8 reason_valid = 1;
7661
7662 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7663 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7664 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7665 /* set status bit */
7666 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7667 }
7668 switch (info) {
7669 case 0:
7670 case 1:
7671 case 2:
7672 case 3:
7673 case 4:
7674 case 5:
7675 case 6:
7676 extra = fm_config_txt[info];
7677 break;
7678 case 8:
7679 extra = fm_config_txt[info];
7680 if (ppd->port_error_action &
7681 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7682 do_bounce = 1;
7683 /*
7684 * lcl_reason cannot be derived from info
7685 * for this error
7686 */
7687 lcl_reason =
7688 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7689 }
7690 break;
7691 default:
7692 reason_valid = 0;
7693 snprintf(buf, sizeof(buf), "reserved%lld", info);
7694 extra = buf;
7695 break;
7696 }
7697
7698 if (reason_valid && !do_bounce) {
7699 do_bounce = ppd->port_error_action &
7700 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7701 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7702 }
7703
7704 /* just report this */
7705 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
7706 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7707 }
7708
7709 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
7710 u8 reason_valid = 1;
7711
7712 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
7713 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
7714 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
7715 if (!(dd->err_info_rcvport.status_and_code &
7716 OPA_EI_STATUS_SMASK)) {
7717 dd->err_info_rcvport.status_and_code =
7718 info & OPA_EI_CODE_SMASK;
7719 /* set status bit */
7720 dd->err_info_rcvport.status_and_code |=
7721 OPA_EI_STATUS_SMASK;
4d114fdd
JJ		 7722 /*
7723 * save first 2 flits in the packet that caused
7724 * the error
7725 */
77241056
MM		 7726 dd->err_info_rcvport.packet_flit1 = hdr0;
7727 dd->err_info_rcvport.packet_flit2 = hdr1;
7728 }
7729 switch (info) {
7730 case 1:
7731 case 2:
7732 case 3:
7733 case 4:
7734 case 5:
7735 case 6:
7736 case 7:
7737 case 9:
7738 case 11:
7739 case 12:
7740 extra = port_rcv_txt[info];
7741 break;
7742 default:
7743 reason_valid = 0;
7744 snprintf(buf, sizeof(buf), "reserved%lld", info);
7745 extra = buf;
7746 break;
7747 }
7748
7749 if (reason_valid && !do_bounce) {
7750 do_bounce = ppd->port_error_action &
7751 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
7752 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
7753 }
7754
7755 /* just report this */
7756 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
7757 dd_dev_info(dd, " hdr0 0x%llx, hdr1 0x%llx\n",
17fb4f29 7758 hdr0, hdr1);
77241056
MM		 7759
7760 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
7761 }
7762
7763 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
7764 /* informative only */
7765 dd_dev_info(dd, "8051 access to LCB blocked\n");
7766 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
7767 }
7768 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
7769 /* informative only */
7770 dd_dev_info(dd, "host access to LCB blocked\n");
7771 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
7772 }
7773
7774 /* report any remaining errors */
7775 if (reg)
7776 dd_dev_info(dd, "DCC Error: %s\n",
17fb4f29 7777 dcc_err_string(buf, sizeof(buf), reg));
77241056
MM		 7778
7779 if (lcl_reason == 0)
7780 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
7781
7782 if (do_bounce) {
7783 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
7784 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
7785 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
7786 }
7787}
7788
7789static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7790{
7791 char buf[96];
7792
7793 dd_dev_info(dd, "LCB Error: %s\n",
17fb4f29 7794 lcb_err_string(buf, sizeof(buf), reg));
77241056
MM		 7795}
7796
7797/*
7798 * CCE block DC interrupt. Source is < 8.
7799 */
7800static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
7801{
7802 const struct err_reg_info *eri = &dc_errs[source];
7803
7804 if (eri->handler) {
7805 interrupt_clear_down(dd, 0, eri);
7806 } else if (source == 3 /* dc_lbm_int */) {
7807 /*
7808 * This indicates that a parity error has occurred on the
7809 * address/control lines presented to the LBM. The error
7810 * is a single pulse, there is no associated error flag,
7811 * and it is non-maskable. This is because if a parity
7812 * error occurs on the request the request is dropped.
7813 * This should never occur, but it is nice to know if it
7814 * ever does.
7815 */
7816 dd_dev_err(dd, "Parity error in DC LBM block\n");
7817 } else {
7818 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
7819 }
7820}
7821
7822/*
7823 * TX block send credit interrupt. Source is < 160.
7824 */
7825static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
7826{
7827 sc_group_release_update(dd, source);
7828}
7829
7830/*
7831 * TX block SDMA interrupt. Source is < 48.
7832 *
7833 * SDMA interrupts are grouped by type:
7834 *
7835 * 0 - N-1 = SDma
7836 * N - 2N-1 = SDmaProgress
7837 * 2N - 3N-1 = SDmaIdle
7838 */
7839static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
7840{
7841 /* what interrupt */
7842 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
7843 /* which engine */
7844 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
7845
7846#ifdef CONFIG_SDMA_VERBOSITY
7847 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
7848 slashstrip(__FILE__), __LINE__, __func__);
7849 sdma_dumpstate(&dd->per_sdma[which]);
7850#endif
7851
7852 if (likely(what < 3 && which < dd->num_sdma)) {
7853 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
7854 } else {
7855 /* should not happen */
7856 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
7857 }
7858}
7859
7860/*
7861 * RX block receive available interrupt. Source is < 160.
7862 */
7863static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
7864{
7865 struct hfi1_ctxtdata *rcd;
7866 char *err_detail;
7867
7868 if (likely(source < dd->num_rcv_contexts)) {
7869 rcd = dd->rcd[source];
7870 if (rcd) {
7871 if (source < dd->first_user_ctxt)
f4f30031 7872 rcd->do_interrupt(rcd, 0);
77241056
MM		 7873 else
7874 handle_user_interrupt(rcd);
7875 return; /* OK */
7876 }
7877 /* received an interrupt, but no rcd */
7878 err_detail = "dataless";
7879 } else {
7880 /* received an interrupt, but are not using that context */
7881 err_detail = "out of range";
7882 }
7883 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
17fb4f29 7884 err_detail, source);
77241056
MM		 7885}
7886
7887/*
7888 * RX block receive urgent interrupt. Source is < 160.
7889 */
7890static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
7891{
7892 struct hfi1_ctxtdata *rcd;
7893 char *err_detail;
7894
7895 if (likely(source < dd->num_rcv_contexts)) {
7896 rcd = dd->rcd[source];
7897 if (rcd) {
7898 /* only pay attention to user urgent interrupts */
7899 if (source >= dd->first_user_ctxt)
7900 handle_user_interrupt(rcd);
7901 return; /* OK */
7902 }
7903 /* received an interrupt, but no rcd */
7904 err_detail = "dataless";
7905 } else {
7906 /* received an interrupt, but are not using that context */
7907 err_detail = "out of range";
7908 }
7909 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
17fb4f29 7910 err_detail, source);
77241056
MM		 7911}
7912
7913/*
7914 * Reserved range interrupt. Should not be called in normal operation.
7915 */
7916static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
7917{
7918 char name[64];
7919
7920 dd_dev_err(dd, "unexpected %s interrupt\n",
17fb4f29 7921 is_reserved_name(name, sizeof(name), source));
77241056
MM		 7922}
7923
7924static const struct is_table is_table[] = {
4d114fdd
JJ		 7925/*
7926 * start end
7927 * name func interrupt func
7928 */
77241056
MM		 7929{ IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
7930 is_misc_err_name, is_misc_err_int },
7931{ IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
7932 is_sdma_eng_err_name, is_sdma_eng_err_int },
7933{ IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
7934 is_sendctxt_err_name, is_sendctxt_err_int },
7935{ IS_SDMA_START, IS_SDMA_END,
7936 is_sdma_eng_name, is_sdma_eng_int },
7937{ IS_VARIOUS_START, IS_VARIOUS_END,
7938 is_various_name, is_various_int },
7939{ IS_DC_START, IS_DC_END,
7940 is_dc_name, is_dc_int },
7941{ IS_RCVAVAIL_START, IS_RCVAVAIL_END,
7942 is_rcv_avail_name, is_rcv_avail_int },
7943{ IS_RCVURGENT_START, IS_RCVURGENT_END,
7944 is_rcv_urgent_name, is_rcv_urgent_int },
7945{ IS_SENDCREDIT_START, IS_SENDCREDIT_END,
7946 is_send_credit_name, is_send_credit_int},
7947{ IS_RESERVED_START, IS_RESERVED_END,
7948 is_reserved_name, is_reserved_int},
7949};
7950
7951/*
7952 * Interrupt source interrupt - called when the given source has an interrupt.
7953 * Source is a bit index into an array of 64-bit integers.
7954 */
7955static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
7956{
7957 const struct is_table *entry;
7958
7959 /* avoids a double compare by walking the table in-order */
7960 for (entry = &is_table[0]; entry->is_name; entry++) {
7961 if (source < entry->end) {
7962 trace_hfi1_interrupt(dd, entry, source);
7963 entry->is_int(dd, source - entry->start);
7964 return;
7965 }
7966 }
7967 /* fell off the end */
7968 dd_dev_err(dd, "invalid interrupt source %u\n", source);
7969}
7970
7971/*
7972 * General interrupt handler. This is able to correctly handle
7973 * all interrupts in case INTx is used.
7974 */
7975static irqreturn_t general_interrupt(int irq, void *data)
7976{
7977 struct hfi1_devdata *dd = data;
7978 u64 regs[CCE_NUM_INT_CSRS];
7979 u32 bit;
7980 int i;
7981
7982 this_cpu_inc(*dd->int_counter);
7983
7984 /* phase 1: scan and clear all handled interrupts */
7985 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
7986 if (dd->gi_mask[i] == 0) {
7987 regs[i] = 0; /* used later */
7988 continue;
7989 }
7990 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
7991 dd->gi_mask[i];
7992 /* only clear if anything is set */
7993 if (regs[i])
7994 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
7995 }
7996
7997 /* phase 2: call the appropriate handler */
7998 for_each_set_bit(bit, (unsigned long *)&regs[0],
17fb4f29 7999 CCE_NUM_INT_CSRS * 64) {
77241056
MM		 8000 is_interrupt(dd, bit);
8001 }
8002
8003 return IRQ_HANDLED;
8004}
8005
8006static irqreturn_t sdma_interrupt(int irq, void *data)
8007{
8008 struct sdma_engine *sde = data;
8009 struct hfi1_devdata *dd = sde->dd;
8010 u64 status;
8011
8012#ifdef CONFIG_SDMA_VERBOSITY
8013 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8014 slashstrip(__FILE__), __LINE__, __func__);
8015 sdma_dumpstate(sde);
8016#endif
8017
8018 this_cpu_inc(*dd->int_counter);
8019
8020 /* This read_csr is really bad in the hot path */
8021 status = read_csr(dd,
17fb4f29
JJ		 8022 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8023 & sde->imask;
77241056
MM		 8024 if (likely(status)) {
8025 /* clear the interrupt(s) */
8026 write_csr(dd,
17fb4f29
JJ		 8027 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8028 status);
77241056
MM		 8029
8030 /* handle the interrupt(s) */
8031 sdma_engine_interrupt(sde, status);
8032 } else
8033 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
17fb4f29 8034 sde->this_idx);
77241056
MM		 8035
8036 return IRQ_HANDLED;
8037}
8038
8039/*
ecd42f8d
DL		 8040 * Clear the receive interrupt. Use a read of the interrupt clear CSR
8041 * to insure that the write completed. This does NOT guarantee that
8042 * queued DMA writes to memory from the chip are pushed.
f4f30031
DL		 8043 */
8044static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8045{
8046 struct hfi1_devdata *dd = rcd->dd;
8047 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8048
8049 mmiowb(); /* make sure everything before is written */
8050 write_csr(dd, addr, rcd->imask);
8051 /* force the above write on the chip and get a value back */
8052 (void)read_csr(dd, addr);
8053}
8054
8055/* force the receive interrupt */
fb9036dd 8056void force_recv_intr(struct hfi1_ctxtdata *rcd)
f4f30031
DL		 8057{
8058 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8059}
8060
ecd42f8d
DL		 8061/*
8062 * Return non-zero if a packet is present.
8063 *
8064 * This routine is called when rechecking for packets after the RcvAvail
8065 * interrupt has been cleared down. First, do a quick check of memory for
8066 * a packet present. If not found, use an expensive CSR read of the context
8067 * tail to determine the actual tail. The CSR read is necessary because there
8068 * is no method to push pending DMAs to memory other than an interrupt and we
8069 * are trying to determine if we need to force an interrupt.
8070 */
f4f30031
DL		 8071static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8072{
ecd42f8d
DL		 8073 u32 tail;
8074 int present;
8075
f4f30031 8076 if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
ecd42f8d 8077 present = (rcd->seq_cnt ==
f4f30031 8078 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
ecd42f8d
DL		 8079 else /* is RDMA rtail */
8080 present = (rcd->head != get_rcvhdrtail(rcd));
8081
8082 if (present)
8083 return 1;
f4f30031 8084
ecd42f8d
DL		 8085 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8086 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8087 return rcd->head != tail;
f4f30031
DL		 8088}
8089
8090/*
8091 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
8092 * This routine will try to handle packets immediately (latency), but if
8093 * it finds too many, it will invoke the thread handler (bandwitdh). The
16733b88 8094 * chip receive interrupt is *not* cleared down until this or the thread (if
f4f30031
DL		 8095 * invoked) is finished. The intent is to avoid extra interrupts while we
8096 * are processing packets anyway.
77241056
MM		 8097 */
8098static irqreturn_t receive_context_interrupt(int irq, void *data)
8099{
8100 struct hfi1_ctxtdata *rcd = data;
8101 struct hfi1_devdata *dd = rcd->dd;
f4f30031
DL		 8102 int disposition;
8103 int present;
77241056
MM		 8104
8105 trace_hfi1_receive_interrupt(dd, rcd->ctxt);
8106 this_cpu_inc(*dd->int_counter);
affa48de 8107 aspm_ctx_disable(rcd);
77241056 8108
f4f30031
DL		 8109 /* receive interrupt remains blocked while processing packets */
8110 disposition = rcd->do_interrupt(rcd, 0);
77241056 8111
f4f30031
DL		 8112 /*
8113 * Too many packets were seen while processing packets in this
8114 * IRQ handler. Invoke the handler thread. The receive interrupt
8115 * remains blocked.
8116 */
8117 if (disposition == RCV_PKT_LIMIT)
8118 return IRQ_WAKE_THREAD;
8119
8120 /*
8121 * The packet processor detected no more packets. Clear the receive
8122 * interrupt and recheck for a packet packet that may have arrived
8123 * after the previous check and interrupt clear. If a packet arrived,
8124 * force another interrupt.
8125 */
8126 clear_recv_intr(rcd);
8127 present = check_packet_present(rcd);
8128 if (present)
8129 force_recv_intr(rcd);
8130
8131 return IRQ_HANDLED;
8132}
8133
8134/*
8135 * Receive packet thread handler. This expects to be invoked with the
8136 * receive interrupt still blocked.
8137 */
8138static irqreturn_t receive_context_thread(int irq, void *data)
8139{
8140 struct hfi1_ctxtdata *rcd = data;
8141 int present;
8142
8143 /* receive interrupt is still blocked from the IRQ handler */
8144 (void)rcd->do_interrupt(rcd, 1);
8145
8146 /*
8147 * The packet processor will only return if it detected no more
8148 * packets. Hold IRQs here so we can safely clear the interrupt and
8149 * recheck for a packet that may have arrived after the previous
8150 * check and the interrupt clear. If a packet arrived, force another
8151 * interrupt.
8152 */
8153 local_irq_disable();
8154 clear_recv_intr(rcd);
8155 present = check_packet_present(rcd);
8156 if (present)
8157 force_recv_intr(rcd);
8158 local_irq_enable();
77241056
MM		 8159
8160 return IRQ_HANDLED;
8161}
8162
8163/* ========================================================================= */
8164
8165u32 read_physical_state(struct hfi1_devdata *dd)
8166{
8167 u64 reg;
8168
8169 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8170 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8171 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8172}
8173
fb9036dd 8174u32 read_logical_state(struct hfi1_devdata *dd)
77241056
MM		 8175{
8176 u64 reg;
8177
8178 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8179 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8180 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8181}
8182
8183static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8184{
8185 u64 reg;
8186
8187 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8188 /* clear current state, set new state */
8189 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8190 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8191 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8192}
8193
8194/*
8195 * Use the 8051 to read a LCB CSR.
8196 */
8197static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8198{
8199 u32 regno;
8200 int ret;
8201
8202 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8203 if (acquire_lcb_access(dd, 0) == 0) {
8204 *data = read_csr(dd, addr);
8205 release_lcb_access(dd, 0);
8206 return 0;
8207 }
8208 return -EBUSY;
8209 }
8210
8211 /* register is an index of LCB registers: (offset - base) / 8 */
8212 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8213 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8214 if (ret != HCMD_SUCCESS)
8215 return -EBUSY;
8216 return 0;
8217}
8218
8219/*
8220 * Read an LCB CSR. Access may not be in host control, so check.
8221 * Return 0 on success, -EBUSY on failure.
8222 */
8223int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8224{
8225 struct hfi1_pportdata *ppd = dd->pport;
8226
8227 /* if up, go through the 8051 for the value */
8228 if (ppd->host_link_state & HLS_UP)
8229 return read_lcb_via_8051(dd, addr, data);
8230 /* if going up or down, no access */
8231 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8232 return -EBUSY;
8233 /* otherwise, host has access */
8234 *data = read_csr(dd, addr);
8235 return 0;
8236}
8237
8238/*
8239 * Use the 8051 to write a LCB CSR.
8240 */
8241static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8242{
3bf40d65
DL		 8243 u32 regno;
8244 int ret;
77241056 8245
3bf40d65
DL		 8246 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8247 (dd->dc8051_ver < dc8051_ver(0, 20))) {
8248 if (acquire_lcb_access(dd, 0) == 0) {
8249 write_csr(dd, addr, data);
8250 release_lcb_access(dd, 0);
8251 return 0;
8252 }
8253 return -EBUSY;
77241056 8254 }
3bf40d65
DL		 8255
8256 /* register is an index of LCB registers: (offset - base) / 8 */
8257 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8258 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8259 if (ret != HCMD_SUCCESS)
8260 return -EBUSY;
8261 return 0;
77241056
MM		 8262}
8263
8264/*
8265 * Write an LCB CSR. Access may not be in host control, so check.
8266 * Return 0 on success, -EBUSY on failure.
8267 */
8268int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8269{
8270 struct hfi1_pportdata *ppd = dd->pport;
8271
8272 /* if up, go through the 8051 for the value */
8273 if (ppd->host_link_state & HLS_UP)
8274 return write_lcb_via_8051(dd, addr, data);
8275 /* if going up or down, no access */
8276 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8277 return -EBUSY;
8278 /* otherwise, host has access */
8279 write_csr(dd, addr, data);
8280 return 0;
8281}
8282
8283/*
8284 * Returns:
8285 * < 0 = Linux error, not able to get access
8286 * > 0 = 8051 command RETURN_CODE
8287 */
8288static int do_8051_command(
8289 struct hfi1_devdata *dd,
8290 u32 type,
8291 u64 in_data,
8292 u64 *out_data)
8293{
8294 u64 reg, completed;
8295 int return_code;
8296 unsigned long flags;
8297 unsigned long timeout;
8298
8299 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8300
8301 /*
8302 * Alternative to holding the lock for a long time:
8303 * - keep busy wait - have other users bounce off
8304 */
8305 spin_lock_irqsave(&dd->dc8051_lock, flags);
8306
8307 /* We can't send any commands to the 8051 if it's in reset */
8308 if (dd->dc_shutdown) {
8309 return_code = -ENODEV;
8310 goto fail;
8311 }
8312
8313 /*
8314 * If an 8051 host command timed out previously, then the 8051 is
8315 * stuck.
8316 *
8317 * On first timeout, attempt to reset and restart the entire DC
8318 * block (including 8051). (Is this too big of a hammer?)
8319 *
8320 * If the 8051 times out a second time, the reset did not bring it
8321 * back to healthy life. In that case, fail any subsequent commands.
8322 */
8323 if (dd->dc8051_timed_out) {
8324 if (dd->dc8051_timed_out > 1) {
8325 dd_dev_err(dd,
8326 "Previous 8051 host command timed out, skipping command %u\n",
8327 type);
8328 return_code = -ENXIO;
8329 goto fail;
8330 }
8331 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8332 dc_shutdown(dd);
8333 dc_start(dd);
8334 spin_lock_irqsave(&dd->dc8051_lock, flags);
8335 }
8336
8337 /*
8338 * If there is no timeout, then the 8051 command interface is
8339 * waiting for a command.
8340 */
8341
3bf40d65
DL		 8342 /*
8343 * When writing a LCB CSR, out_data contains the full value to
8344 * to be written, while in_data contains the relative LCB
8345 * address in 7:0. Do the work here, rather than the caller,
8346 * of distrubting the write data to where it needs to go:
8347 *
8348 * Write data
8349 * 39:00 -> in_data[47:8]
8350 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8351 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8352 */
8353 if (type == HCMD_WRITE_LCB_CSR) {
8354 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8355 reg = ((((*out_data) >> 40) & 0xff) <<
8356 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8357 | ((((*out_data) >> 48) & 0xffff) <<
8358 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8359 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8360 }
8361
77241056
MM		 8362 /*
8363 * Do two writes: the first to stabilize the type and req_data, the
8364 * second to activate.
8365 */
8366 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8367 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8368 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8369 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8370 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8371 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8372 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8373
8374 /* wait for completion, alternate: interrupt */
8375 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8376 while (1) {
8377 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8378 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8379 if (completed)
8380 break;
8381 if (time_after(jiffies, timeout)) {
8382 dd->dc8051_timed_out++;
8383 dd_dev_err(dd, "8051 host command %u timeout\n", type);
8384 if (out_data)
8385 *out_data = 0;
8386 return_code = -ETIMEDOUT;
8387 goto fail;
8388 }
8389 udelay(2);
8390 }
8391
8392 if (out_data) {
8393 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8394 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8395 if (type == HCMD_READ_LCB_CSR) {
8396 /* top 16 bits are in a different register */
8397 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8398 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8399 << (48
8400 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8401 }
8402 }
8403 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8404 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8405 dd->dc8051_timed_out = 0;
8406 /*
8407 * Clear command for next user.
8408 */
8409 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8410
8411fail:
8412 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8413
8414 return return_code;
8415}
8416
8417static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8418{
8419 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8420}
8421
8ebd4cf1
EH		 8422int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8423 u8 lane_id, u32 config_data)
77241056
MM		 8424{
8425 u64 data;
8426 int ret;
8427
8428 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8429 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8430 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8431 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8432 if (ret != HCMD_SUCCESS) {
8433 dd_dev_err(dd,
17fb4f29
JJ		 8434 "load 8051 config: field id %d, lane %d, err %d\n",
8435 (int)field_id, (int)lane_id, ret);
77241056
MM		 8436 }
8437 return ret;
8438}
8439
8440/*
8441 * Read the 8051 firmware "registers". Use the RAM directly. Always
8442 * set the result, even on error.
8443 * Return 0 on success, -errno on failure
8444 */
8ebd4cf1
EH		 8445int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8446 u32 *result)
77241056
MM		 8447{
8448 u64 big_data;
8449 u32 addr;
8450 int ret;
8451
8452 /* address start depends on the lane_id */
8453 if (lane_id < 4)
8454 addr = (4 * NUM_GENERAL_FIELDS)
8455 + (lane_id * 4 * NUM_LANE_FIELDS);
8456 else
8457 addr = 0;
8458 addr += field_id * 4;
8459
8460 /* read is in 8-byte chunks, hardware will truncate the address down */
8461 ret = read_8051_data(dd, addr, 8, &big_data);
8462
8463 if (ret == 0) {
8464 /* extract the 4 bytes we want */
8465 if (addr & 0x4)
8466 *result = (u32)(big_data >> 32);
8467 else
8468 *result = (u32)big_data;
8469 } else {
8470 *result = 0;
8471 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
17fb4f29 8472 __func__, lane_id, field_id);
77241056
MM		 8473 }
8474
8475 return ret;
8476}
8477
8478static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8479 u8 continuous)
8480{
8481 u32 frame;
8482
8483 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8484 | power_management << POWER_MANAGEMENT_SHIFT;
8485 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8486 GENERAL_CONFIG, frame);
8487}
8488
8489static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8490 u16 vl15buf, u8 crc_sizes)
8491{
8492 u32 frame;
8493
8494 frame = (u32)vau << VAU_SHIFT
8495 | (u32)z << Z_SHIFT
8496 | (u32)vcu << VCU_SHIFT
8497 | (u32)vl15buf << VL15BUF_SHIFT
8498 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8499 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8500 GENERAL_CONFIG, frame);
8501}
8502
8503static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
8504 u8 *flag_bits, u16 *link_widths)
8505{
8506 u32 frame;
8507
8508 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
17fb4f29 8509 &frame);
77241056
MM		 8510 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8511 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8512 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8513}
8514
8515static int write_vc_local_link_width(struct hfi1_devdata *dd,
8516 u8 misc_bits,
8517 u8 flag_bits,
8518 u16 link_widths)
8519{
8520 u32 frame;
8521
8522 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8523 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8524 | (u32)link_widths << LINK_WIDTH_SHIFT;
8525 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8526 frame);
8527}
8528
8529static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8530 u8 device_rev)
8531{
8532 u32 frame;
8533
8534 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8535 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8536 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8537}
8538
8539static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8540 u8 *device_rev)
8541{
8542 u32 frame;
8543
8544 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8545 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8546 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8547 & REMOTE_DEVICE_REV_MASK;
8548}
8549
8550void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
8551{
8552 u32 frame;
8553
8554 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8555 *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
8556 *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
8557}
8558
8559static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8560 u8 *continuous)
8561{
8562 u32 frame;
8563
8564 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8565 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8566 & POWER_MANAGEMENT_MASK;
8567 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8568 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8569}
8570
8571static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8572 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8573{
8574 u32 frame;
8575
8576 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8577 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8578 *z = (frame >> Z_SHIFT) & Z_MASK;
8579 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8580 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8581 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8582}
8583
8584static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8585 u8 *remote_tx_rate,
8586 u16 *link_widths)
8587{
8588 u32 frame;
8589
8590 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
17fb4f29 8591 &frame);
77241056
MM		 8592 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8593 & REMOTE_TX_RATE_MASK;
8594 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8595}
8596
8597static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8598{
8599 u32 frame;
8600
8601 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8602 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8603}
8604
8605static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
8606{
8607 u32 frame;
8608
8609 read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
8610 *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
8611}
8612
8613static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8614{
8615 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8616}
8617
8618static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8619{
8620 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8621}
8622
8623void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
8624{
8625 u32 frame;
8626 int ret;
8627
8628 *link_quality = 0;
8629 if (dd->pport->host_link_state & HLS_UP) {
8630 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
17fb4f29 8631 &frame);
77241056
MM		 8632 if (ret == 0)
8633 *link_quality = (frame >> LINK_QUALITY_SHIFT)
8634 & LINK_QUALITY_MASK;
8635 }
8636}
8637
8638static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
8639{
8640 u32 frame;
8641
8642 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
8643 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
8644}
8645
8646static int read_tx_settings(struct hfi1_devdata *dd,
8647 u8 *enable_lane_tx,
8648 u8 *tx_polarity_inversion,
8649 u8 *rx_polarity_inversion,
8650 u8 *max_rate)
8651{
8652 u32 frame;
8653 int ret;
8654
8655 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
8656 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
8657 & ENABLE_LANE_TX_MASK;
8658 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
8659 & TX_POLARITY_INVERSION_MASK;
8660 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
8661 & RX_POLARITY_INVERSION_MASK;
8662 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
8663 return ret;
8664}
8665
8666static int write_tx_settings(struct hfi1_devdata *dd,
8667 u8 enable_lane_tx,
8668 u8 tx_polarity_inversion,
8669 u8 rx_polarity_inversion,
8670 u8 max_rate)
8671{
8672 u32 frame;
8673
8674 /* no need to mask, all variable sizes match field widths */
8675 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
8676 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
8677 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
8678 | max_rate << MAX_RATE_SHIFT;
8679 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
8680}
8681
8682static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
8683{
8684 u32 frame, version, prod_id;
8685 int ret, lane;
8686
8687 /* 4 lanes */
8688 for (lane = 0; lane < 4; lane++) {
8689 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
8690 if (ret) {
17fb4f29
JJ		 8691 dd_dev_err(dd,
8692 "Unable to read lane %d firmware details\n",
8693 lane);
77241056
MM		 8694 continue;
8695 }
8696 version = (frame >> SPICO_ROM_VERSION_SHIFT)
8697 & SPICO_ROM_VERSION_MASK;
8698 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
8699 & SPICO_ROM_PROD_ID_MASK;
8700 dd_dev_info(dd,
17fb4f29
JJ		 8701 "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
8702 lane, version, prod_id);
77241056
MM		 8703 }
8704}
8705
8706/*
8707 * Read an idle LCB message.
8708 *
8709 * Returns 0 on success, -EINVAL on error
8710 */
8711static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
8712{
8713 int ret;
8714
17fb4f29 8715 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
77241056
MM		 8716 if (ret != HCMD_SUCCESS) {
8717 dd_dev_err(dd, "read idle message: type %d, err %d\n",
17fb4f29 8718 (u32)type, ret);
77241056
MM		 8719 return -EINVAL;
8720 }
8721 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
8722 /* return only the payload as we already know the type */
8723 *data_out >>= IDLE_PAYLOAD_SHIFT;
8724 return 0;
8725}
8726
8727/*
8728 * Read an idle SMA message. To be done in response to a notification from
8729 * the 8051.
8730 *
8731 * Returns 0 on success, -EINVAL on error
8732 */
8733static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
8734{
17fb4f29
JJ		 8735 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
8736 data);
77241056
MM		 8737}
8738
8739/*
8740 * Send an idle LCB message.
8741 *
8742 * Returns 0 on success, -EINVAL on error
8743 */
8744static int send_idle_message(struct hfi1_devdata *dd, u64 data)
8745{
8746 int ret;
8747
8748 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
8749 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
8750 if (ret != HCMD_SUCCESS) {
8751 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
17fb4f29 8752 data, ret);
77241056
MM		 8753 return -EINVAL;
8754 }
8755 return 0;
8756}
8757
8758/*
8759 * Send an idle SMA message.
8760 *
8761 * Returns 0 on success, -EINVAL on error
8762 */
8763int send_idle_sma(struct hfi1_devdata *dd, u64 message)
8764{
8765 u64 data;
8766
17fb4f29
JJ		 8767 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
8768 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
77241056
MM		 8769 return send_idle_message(dd, data);
8770}
8771
8772/*
8773 * Initialize the LCB then do a quick link up. This may or may not be
8774 * in loopback.
8775 *
8776 * return 0 on success, -errno on error
8777 */
8778static int do_quick_linkup(struct hfi1_devdata *dd)
8779{
8780 u64 reg;
8781 unsigned long timeout;
8782 int ret;
8783
8784 lcb_shutdown(dd, 0);
8785
8786 if (loopback) {
8787 /* LCB_CFG_LOOPBACK.VAL = 2 */
8788 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
8789 write_csr(dd, DC_LCB_CFG_LOOPBACK,
17fb4f29 8790 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
77241056
MM		 8791 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
8792 }
8793
8794 /* start the LCBs */
8795 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
8796 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
8797
8798 /* simulator only loopback steps */
8799 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8800 /* LCB_CFG_RUN.EN = 1 */
8801 write_csr(dd, DC_LCB_CFG_RUN,
17fb4f29 8802 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
77241056
MM		 8803
8804 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
8805 timeout = jiffies + msecs_to_jiffies(10);
8806 while (1) {
17fb4f29 8807 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
77241056
MM		 8808 if (reg)
8809 break;
8810 if (time_after(jiffies, timeout)) {
8811 dd_dev_err(dd,
17fb4f29 8812 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
77241056
MM		 8813 return -ETIMEDOUT;
8814 }
8815 udelay(2);
8816 }
8817
8818 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
17fb4f29 8819 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
77241056
MM		 8820 }
8821
8822 if (!loopback) {
8823 /*
8824 * When doing quick linkup and not in loopback, both
8825 * sides must be done with LCB set-up before either
8826 * starts the quick linkup. Put a delay here so that
8827 * both sides can be started and have a chance to be
8828 * done with LCB set up before resuming.
8829 */
8830 dd_dev_err(dd,
17fb4f29 8831 "Pausing for peer to be finished with LCB set up\n");
77241056 8832 msleep(5000);
17fb4f29 8833 dd_dev_err(dd, "Continuing with quick linkup\n");
77241056
MM		 8834 }
8835
8836 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
8837 set_8051_lcb_access(dd);
8838
8839 /*
8840 * State "quick" LinkUp request sets the physical link state to
8841 * LinkUp without a verify capability sequence.
8842 * This state is in simulator v37 and later.
8843 */
8844 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
8845 if (ret != HCMD_SUCCESS) {
8846 dd_dev_err(dd,
17fb4f29
JJ		 8847 "%s: set physical link state to quick LinkUp failed with return %d\n",
8848 __func__, ret);
77241056
MM		 8849
8850 set_host_lcb_access(dd);
8851 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
8852
8853 if (ret >= 0)
8854 ret = -EINVAL;
8855 return ret;
8856 }
8857
8858 return 0; /* success */
8859}
8860
8861/*
8862 * Set the SerDes to internal loopback mode.
8863 * Returns 0 on success, -errno on error.
8864 */
8865static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
8866{
8867 int ret;
8868
8869 ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
8870 if (ret == HCMD_SUCCESS)
8871 return 0;
8872 dd_dev_err(dd,
17fb4f29
JJ		 8873 "Set physical link state to SerDes Loopback failed with return %d\n",
8874 ret);
77241056
MM		 8875 if (ret >= 0)
8876 ret = -EINVAL;
8877 return ret;
8878}
8879
8880/*
8881 * Do all special steps to set up loopback.
8882 */
8883static int init_loopback(struct hfi1_devdata *dd)
8884{
8885 dd_dev_info(dd, "Entering loopback mode\n");
8886
8887 /* all loopbacks should disable self GUID check */
8888 write_csr(dd, DC_DC8051_CFG_MODE,
17fb4f29 8889 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
77241056
MM		 8890
8891 /*
8892 * The simulator has only one loopback option - LCB. Switch
8893 * to that option, which includes quick link up.
8894 *
8895 * Accept all valid loopback values.
8896 */
d0d236ea
JJ		 8897 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
8898 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
8899 loopback == LOOPBACK_CABLE)) {
77241056
MM		 8900 loopback = LOOPBACK_LCB;
8901 quick_linkup = 1;
8902 return 0;
8903 }
8904
8905 /* handle serdes loopback */
8906 if (loopback == LOOPBACK_SERDES) {
8907 /* internal serdes loopack needs quick linkup on RTL */
8908 if (dd->icode == ICODE_RTL_SILICON)
8909 quick_linkup = 1;
8910 return set_serdes_loopback_mode(dd);
8911 }
8912
8913 /* LCB loopback - handled at poll time */
8914 if (loopback == LOOPBACK_LCB) {
8915 quick_linkup = 1; /* LCB is always quick linkup */
8916
8917 /* not supported in emulation due to emulation RTL changes */
8918 if (dd->icode == ICODE_FPGA_EMULATION) {
8919 dd_dev_err(dd,
17fb4f29 8920 "LCB loopback not supported in emulation\n");
77241056
MM		 8921 return -EINVAL;
8922 }
8923 return 0;
8924 }
8925
8926 /* external cable loopback requires no extra steps */
8927 if (loopback == LOOPBACK_CABLE)
8928 return 0;
8929
8930 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
8931 return -EINVAL;
8932}
8933
8934/*
8935 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
8936 * used in the Verify Capability link width attribute.
8937 */
8938static u16 opa_to_vc_link_widths(u16 opa_widths)
8939{
8940 int i;
8941 u16 result = 0;
8942
8943 static const struct link_bits {
8944 u16 from;
8945 u16 to;
8946 } opa_link_xlate[] = {
8638b77f
JJ		 8947 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) },
8948 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) },
8949 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) },
8950 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) },
77241056
MM		 8951 };
8952
8953 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
8954 if (opa_widths & opa_link_xlate[i].from)
8955 result |= opa_link_xlate[i].to;
8956 }
8957 return result;
8958}
8959
8960/*
8961 * Set link attributes before moving to polling.
8962 */
8963static int set_local_link_attributes(struct hfi1_pportdata *ppd)
8964{
8965 struct hfi1_devdata *dd = ppd->dd;
8966 u8 enable_lane_tx;
8967 u8 tx_polarity_inversion;
8968 u8 rx_polarity_inversion;
8969 int ret;
8970
8971 /* reset our fabric serdes to clear any lingering problems */
8972 fabric_serdes_reset(dd);
8973
8974 /* set the local tx rate - need to read-modify-write */
8975 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
17fb4f29 8976 &rx_polarity_inversion, &ppd->local_tx_rate);
77241056
MM		 8977 if (ret)
8978 goto set_local_link_attributes_fail;
8979
8980 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
8981 /* set the tx rate to the fastest enabled */
8982 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
8983 ppd->local_tx_rate = 1;
8984 else
8985 ppd->local_tx_rate = 0;
8986 } else {
8987 /* set the tx rate to all enabled */
8988 ppd->local_tx_rate = 0;
8989 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
8990 ppd->local_tx_rate |= 2;
8991 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
8992 ppd->local_tx_rate |= 1;
8993 }
febffe2c
EH		 8994
8995 enable_lane_tx = 0xF; /* enable all four lanes */
77241056 8996 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
17fb4f29 8997 rx_polarity_inversion, ppd->local_tx_rate);
77241056
MM		 8998 if (ret != HCMD_SUCCESS)
8999 goto set_local_link_attributes_fail;
9000
9001 /*
9002 * DC supports continuous updates.
9003 */
17fb4f29
JJ		 9004 ret = write_vc_local_phy(dd,
9005 0 /* no power management */,
9006 1 /* continuous updates */);
77241056
MM		 9007 if (ret != HCMD_SUCCESS)
9008 goto set_local_link_attributes_fail;
9009
9010 /* z=1 in the next call: AU of 0 is not supported by the hardware */
9011 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9012 ppd->port_crc_mode_enabled);
9013 if (ret != HCMD_SUCCESS)
9014 goto set_local_link_attributes_fail;
9015
9016 ret = write_vc_local_link_width(dd, 0, 0,
17fb4f29
JJ		 9017 opa_to_vc_link_widths(
9018 ppd->link_width_enabled));
77241056
MM		 9019 if (ret != HCMD_SUCCESS)
9020 goto set_local_link_attributes_fail;
9021
9022 /* let peer know who we are */
9023 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9024 if (ret == HCMD_SUCCESS)
9025 return 0;
9026
9027set_local_link_attributes_fail:
9028 dd_dev_err(dd,
17fb4f29
JJ		 9029 "Failed to set local link attributes, return 0x%x\n",
9030 ret);
77241056
MM		 9031 return ret;
9032}
9033
9034/*
9035 * Call this to start the link. Schedule a retry if the cable is not
9036 * present or if unable to start polling. Do not do anything if the
9037 * link is disabled. Returns 0 if link is disabled or moved to polling
9038 */
9039int start_link(struct hfi1_pportdata *ppd)
9040{
9041 if (!ppd->link_enabled) {
9042 dd_dev_info(ppd->dd,
17fb4f29
JJ		 9043 "%s: stopping link start because link is disabled\n",
9044 __func__);
77241056
MM		 9045 return 0;
9046 }
9047 if (!ppd->driver_link_ready) {
9048 dd_dev_info(ppd->dd,
17fb4f29
JJ		 9049 "%s: stopping link start because driver is not ready\n",
9050 __func__);
77241056
MM		 9051 return 0;
9052 }
9053
9054 if (qsfp_mod_present(ppd) || loopback == LOOPBACK_SERDES ||
17fb4f29
JJ		 9055 loopback == LOOPBACK_LCB ||
9056 ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
77241056
MM		 9057 return set_link_state(ppd, HLS_DN_POLL);
9058
9059 dd_dev_info(ppd->dd,
17fb4f29
JJ		 9060 "%s: stopping link start because no cable is present\n",
9061 __func__);
77241056
MM		 9062 return -EAGAIN;
9063}
9064
8ebd4cf1
EH		 9065static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9066{
9067 struct hfi1_devdata *dd = ppd->dd;
9068 u64 mask;
9069 unsigned long timeout;
9070
9071 /*
9072 * Check for QSFP interrupt for t_init (SFF 8679)
9073 */
9074 timeout = jiffies + msecs_to_jiffies(2000);
9075 while (1) {
9076 mask = read_csr(dd, dd->hfi1_id ?
9077 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9078 if (!(mask & QSFP_HFI0_INT_N)) {
9079 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR :
9080 ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N);
9081 break;
9082 }
9083 if (time_after(jiffies, timeout)) {
9084 dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9085 __func__);
9086 break;
9087 }
9088 udelay(2);
9089 }
9090}
9091
9092static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9093{
9094 struct hfi1_devdata *dd = ppd->dd;
9095 u64 mask;
9096
9097 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9098 if (enable)
9099 mask |= (u64)QSFP_HFI0_INT_N;
9100 else
9101 mask &= ~(u64)QSFP_HFI0_INT_N;
9102 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9103}
9104
9105void reset_qsfp(struct hfi1_pportdata *ppd)
77241056
MM		 9106{
9107 struct hfi1_devdata *dd = ppd->dd;
9108 u64 mask, qsfp_mask;
9109
8ebd4cf1
EH		 9110 /* Disable INT_N from triggering QSFP interrupts */
9111 set_qsfp_int_n(ppd, 0);
9112
9113 /* Reset the QSFP */
77241056 9114 mask = (u64)QSFP_HFI0_RESET_N;
8ebd4cf1 9115 qsfp_mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE);
77241056 9116 qsfp_mask |= mask;
17fb4f29 9117 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE, qsfp_mask);
77241056 9118
17fb4f29
JJ		 9119 qsfp_mask = read_csr(dd,
9120 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
77241056
MM		 9121 qsfp_mask &= ~mask;
9122 write_csr(dd,
17fb4f29 9123 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
77241056
MM		 9124
9125 udelay(10);
9126
9127 qsfp_mask |= mask;
9128 write_csr(dd,
17fb4f29 9129 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
8ebd4cf1
EH		 9130
9131 wait_for_qsfp_init(ppd);
9132
9133 /*
9134 * Allow INT_N to trigger the QSFP interrupt to watch
9135 * for alarms and warnings
9136 */
9137 set_qsfp_int_n(ppd, 1);
77241056
MM		 9138}
9139
9140static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9141 u8 *qsfp_interrupt_status)
9142{
9143 struct hfi1_devdata *dd = ppd->dd;
9144
9145 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
17fb4f29
JJ		 9146 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9147 dd_dev_info(dd, "%s: QSFP cable on fire\n",
9148 __func__);
77241056
MM		 9149
9150 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
17fb4f29
JJ		 9151 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9152 dd_dev_info(dd, "%s: QSFP cable temperature too low\n",
9153 __func__);
77241056
MM		 9154
9155 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
17fb4f29
JJ		 9156 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9157 dd_dev_info(dd, "%s: QSFP supply voltage too high\n",
9158 __func__);
77241056
MM		 9159
9160 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
17fb4f29
JJ		 9161 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9162 dd_dev_info(dd, "%s: QSFP supply voltage too low\n",
9163 __func__);
77241056
MM		 9164
9165 /* Byte 2 is vendor specific */
9166
9167 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9168 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9169 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n",
9170 __func__);
77241056
MM		 9171
9172 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9173 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9174 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n",
9175 __func__);
77241056
MM		 9176
9177 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9178 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9179 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n",
9180 __func__);
77241056
MM		 9181
9182 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9183 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9184 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n",
9185 __func__);
77241056
MM		 9186
9187 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
17fb4f29
JJ		 9188 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9189 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n",
9190 __func__);
77241056
MM		 9191
9192 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
17fb4f29
JJ		 9193 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9194 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n",
9195 __func__);
77241056
MM		 9196
9197 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
17fb4f29
JJ		 9198 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9199 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n",
9200 __func__);
77241056
MM		 9201
9202 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
17fb4f29
JJ		 9203 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9204 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n",
9205 __func__);
77241056
MM		 9206
9207 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9208 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9209 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n",
9210 __func__);
77241056
MM		 9211
9212 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9213 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9214 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n",
9215 __func__);
77241056
MM		 9216
9217 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9218 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9219 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n",
9220 __func__);
77241056
MM		 9221
9222 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9223 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9224 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n",
9225 __func__);
77241056
MM		 9226
9227 /* Bytes 9-10 and 11-12 are reserved */
9228 /* Bytes 13-15 are vendor specific */
9229
9230 return 0;
9231}
9232
77241056 9233/* This routine will only be scheduled if the QSFP module is present */
8ebd4cf1 9234void qsfp_event(struct work_struct *work)
77241056
MM		 9235{
9236 struct qsfp_data *qd;
9237 struct hfi1_pportdata *ppd;
9238 struct hfi1_devdata *dd;
9239
9240 qd = container_of(work, struct qsfp_data, qsfp_work);
9241 ppd = qd->ppd;
9242 dd = ppd->dd;
9243
9244 /* Sanity check */
9245 if (!qsfp_mod_present(ppd))
9246 return;
9247
9248 /*
9249 * Turn DC back on after cables has been
9250 * re-inserted. Up until now, the DC has been in
9251 * reset to save power.
9252 */
9253 dc_start(dd);
9254
9255 if (qd->cache_refresh_required) {
8ebd4cf1
EH		 9256 set_qsfp_int_n(ppd, 0);
9257
9258 wait_for_qsfp_init(ppd);
9259
9260 /*
9261 * Allow INT_N to trigger the QSFP interrupt to watch
9262 * for alarms and warnings
77241056 9263 */
8ebd4cf1
EH		 9264 set_qsfp_int_n(ppd, 1);
9265
9266 tune_serdes(ppd);
9267
9268 start_link(ppd);
77241056
MM		 9269 }
9270
9271 if (qd->check_interrupt_flags) {
9272 u8 qsfp_interrupt_status[16] = {0,};
9273
9274 if (qsfp_read(ppd, dd->hfi1_id, 6,
9275 &qsfp_interrupt_status[0], 16) != 16) {
9276 dd_dev_info(dd,
17fb4f29
JJ		 9277 "%s: Failed to read status of QSFP module\n",
9278 __func__);
77241056
MM		 9279 } else {
9280 unsigned long flags;
77241056 9281
8ebd4cf1
EH		 9282 handle_qsfp_error_conditions(
9283 ppd, qsfp_interrupt_status);
77241056
MM		 9284 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9285 ppd->qsfp_info.check_interrupt_flags = 0;
9286 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 9287 flags);
77241056
MM		 9288 }
9289 }
9290}
9291
8ebd4cf1 9292static void init_qsfp_int(struct hfi1_devdata *dd)
77241056 9293{
8ebd4cf1
EH		 9294 struct hfi1_pportdata *ppd = dd->pport;
9295 u64 qsfp_mask, cce_int_mask;
9296 const int qsfp1_int_smask = QSFP1_INT % 64;
9297 const int qsfp2_int_smask = QSFP2_INT % 64;
77241056 9298
8ebd4cf1
EH		 9299 /*
9300 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
9301 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
9302 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
9303 * the index of the appropriate CSR in the CCEIntMask CSR array
9304 */
9305 cce_int_mask = read_csr(dd, CCE_INT_MASK +
9306 (8 * (QSFP1_INT / 64)));
9307 if (dd->hfi1_id) {
9308 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
9309 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)),
9310 cce_int_mask);
9311 } else {
9312 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
9313 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)),
9314 cce_int_mask);
77241056
MM		 9315 }
9316
77241056
MM		 9317 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9318 /* Clear current status to avoid spurious interrupts */
8ebd4cf1
EH		 9319 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9320 qsfp_mask);
9321 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9322 qsfp_mask);
9323
9324 set_qsfp_int_n(ppd, 0);
77241056
MM		 9325
9326 /* Handle active low nature of INT_N and MODPRST_N pins */
9327 if (qsfp_mod_present(ppd))
9328 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9329 write_csr(dd,
9330 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9331 qsfp_mask);
77241056
MM		 9332}
9333
bbdeb33d
DL		 9334/*
9335 * Do a one-time initialize of the LCB block.
9336 */
9337static void init_lcb(struct hfi1_devdata *dd)
9338{
a59329d5
DL		 9339 /* simulator does not correctly handle LCB cclk loopback, skip */
9340 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9341 return;
9342
bbdeb33d
DL		 9343 /* the DC has been reset earlier in the driver load */
9344
9345 /* set LCB for cclk loopback on the port */
9346 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9347 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9348 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9349 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9350 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9351 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9352 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9353}
9354
77241056
MM		 9355int bringup_serdes(struct hfi1_pportdata *ppd)
9356{
9357 struct hfi1_devdata *dd = ppd->dd;
9358 u64 guid;
9359 int ret;
9360
9361 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9362 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9363
9364 guid = ppd->guid;
9365 if (!guid) {
9366 if (dd->base_guid)
9367 guid = dd->base_guid + ppd->port - 1;
9368 ppd->guid = guid;
9369 }
9370
77241056
MM		 9371 /* Set linkinit_reason on power up per OPA spec */
9372 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9373
bbdeb33d
DL		 9374 /* one-time init of the LCB */
9375 init_lcb(dd);
9376
77241056
MM		 9377 if (loopback) {
9378 ret = init_loopback(dd);
9379 if (ret < 0)
9380 return ret;
9381 }
9382
8ebd4cf1
EH		 9383 /* tune the SERDES to a ballpark setting for
9384 * optimal signal and bit error rate
9385 * Needs to be done before starting the link
9386 */
9387 tune_serdes(ppd);
9388
77241056
MM		 9389 return start_link(ppd);
9390}
9391
9392void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9393{
9394 struct hfi1_devdata *dd = ppd->dd;
9395
9396 /*
9397 * Shut down the link and keep it down. First turn off that the
9398 * driver wants to allow the link to be up (driver_link_ready).
9399 * Then make sure the link is not automatically restarted
9400 * (link_enabled). Cancel any pending restart. And finally
9401 * go offline.
9402 */
9403 ppd->driver_link_ready = 0;
9404 ppd->link_enabled = 0;
9405
8ebd4cf1
EH		 9406 ppd->offline_disabled_reason =
9407 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED);
77241056 9408 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
17fb4f29 9409 OPA_LINKDOWN_REASON_SMA_DISABLED);
77241056
MM		 9410 set_link_state(ppd, HLS_DN_OFFLINE);
9411
9412 /* disable the port */
9413 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9414}
9415
9416static inline int init_cpu_counters(struct hfi1_devdata *dd)
9417{
9418 struct hfi1_pportdata *ppd;
9419 int i;
9420
9421 ppd = (struct hfi1_pportdata *)(dd + 1);
9422 for (i = 0; i < dd->num_pports; i++, ppd++) {
4eb06882
DD		 9423 ppd->ibport_data.rvp.rc_acks = NULL;
9424 ppd->ibport_data.rvp.rc_qacks = NULL;
9425 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9426 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9427 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9428 if (!ppd->ibport_data.rvp.rc_acks ||
9429 !ppd->ibport_data.rvp.rc_delayed_comp ||
9430 !ppd->ibport_data.rvp.rc_qacks)
77241056
MM		 9431 return -ENOMEM;
9432 }
9433
9434 return 0;
9435}
9436
9437static const char * const pt_names[] = {
9438 "expected",
9439 "eager",
9440 "invalid"
9441};
9442
9443static const char *pt_name(u32 type)
9444{
9445 return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
9446}
9447
9448/*
9449 * index is the index into the receive array
9450 */
9451void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9452 u32 type, unsigned long pa, u16 order)
9453{
9454 u64 reg;
9455 void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
9456 (dd->kregbase + RCV_ARRAY));
9457
9458 if (!(dd->flags & HFI1_PRESENT))
9459 goto done;
9460
9461 if (type == PT_INVALID) {
9462 pa = 0;
9463 } else if (type > PT_INVALID) {
9464 dd_dev_err(dd,
17fb4f29
JJ		 9465 "unexpected receive array type %u for index %u, not handled\n",
9466 type, index);
77241056
MM		 9467 goto done;
9468 }
9469
9470 hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
9471 pt_name(type), index, pa, (unsigned long)order);
9472
9473#define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
9474 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9475 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9476 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9477 << RCV_ARRAY_RT_ADDR_SHIFT;
9478 writeq(reg, base + (index * 8));
9479
9480 if (type == PT_EAGER)
9481 /*
9482 * Eager entries are written one-by-one so we have to push them
9483 * after we write the entry.
9484 */
9485 flush_wc();
9486done:
9487 return;
9488}
9489
9490void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9491{
9492 struct hfi1_devdata *dd = rcd->dd;
9493 u32 i;
9494
9495 /* this could be optimized */
9496 for (i = rcd->eager_base; i < rcd->eager_base +
9497 rcd->egrbufs.alloced; i++)
9498 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9499
9500 for (i = rcd->expected_base;
9501 i < rcd->expected_base + rcd->expected_count; i++)
9502 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9503}
9504
9505int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
9506 struct hfi1_ctxt_info *kinfo)
9507{
9508 kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
9509 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
9510 return 0;
9511}
9512
9513struct hfi1_message_header *hfi1_get_msgheader(
9514 struct hfi1_devdata *dd, __le32 *rhf_addr)
9515{
9516 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
9517
9518 return (struct hfi1_message_header *)
9519 (rhf_addr - dd->rhf_offset + offset);
9520}
9521
9522static const char * const ib_cfg_name_strings[] = {
9523 "HFI1_IB_CFG_LIDLMC",
9524 "HFI1_IB_CFG_LWID_DG_ENB",
9525 "HFI1_IB_CFG_LWID_ENB",
9526 "HFI1_IB_CFG_LWID",
9527 "HFI1_IB_CFG_SPD_ENB",
9528 "HFI1_IB_CFG_SPD",
9529 "HFI1_IB_CFG_RXPOL_ENB",
9530 "HFI1_IB_CFG_LREV_ENB",
9531 "HFI1_IB_CFG_LINKLATENCY",
9532 "HFI1_IB_CFG_HRTBT",
9533 "HFI1_IB_CFG_OP_VLS",
9534 "HFI1_IB_CFG_VL_HIGH_CAP",
9535 "HFI1_IB_CFG_VL_LOW_CAP",
9536 "HFI1_IB_CFG_OVERRUN_THRESH",
9537 "HFI1_IB_CFG_PHYERR_THRESH",
9538 "HFI1_IB_CFG_LINKDEFAULT",
9539 "HFI1_IB_CFG_PKEYS",
9540 "HFI1_IB_CFG_MTU",
9541 "HFI1_IB_CFG_LSTATE",
9542 "HFI1_IB_CFG_VL_HIGH_LIMIT",
9543 "HFI1_IB_CFG_PMA_TICKS",
9544 "HFI1_IB_CFG_PORT"
9545};
9546
9547static const char *ib_cfg_name(int which)
9548{
9549 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9550 return "invalid";
9551 return ib_cfg_name_strings[which];
9552}
9553
9554int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9555{
9556 struct hfi1_devdata *dd = ppd->dd;
9557 int val = 0;
9558
9559 switch (which) {
9560 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
9561 val = ppd->link_width_enabled;
9562 break;
9563 case HFI1_IB_CFG_LWID: /* currently active Link-width */
9564 val = ppd->link_width_active;
9565 break;
9566 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
9567 val = ppd->link_speed_enabled;
9568 break;
9569 case HFI1_IB_CFG_SPD: /* current Link speed */
9570 val = ppd->link_speed_active;
9571 break;
9572
9573 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
9574 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
9575 case HFI1_IB_CFG_LINKLATENCY:
9576 goto unimplemented;
9577
9578 case HFI1_IB_CFG_OP_VLS:
9579 val = ppd->vls_operational;
9580 break;
9581 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
9582 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9583 break;
9584 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
9585 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9586 break;
9587 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
9588 val = ppd->overrun_threshold;
9589 break;
9590 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
9591 val = ppd->phy_error_threshold;
9592 break;
9593 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
9594 val = dd->link_default;
9595 break;
9596
9597 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
9598 case HFI1_IB_CFG_PMA_TICKS:
9599 default:
9600unimplemented:
9601 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
9602 dd_dev_info(
9603 dd,
9604 "%s: which %s: not implemented\n",
9605 __func__,
9606 ib_cfg_name(which));
9607 break;
9608 }
9609
9610 return val;
9611}
9612
9613/*
9614 * The largest MAD packet size.
9615 */
9616#define MAX_MAD_PACKET 2048
9617
9618/*
9619 * Return the maximum header bytes that can go on the _wire_
9620 * for this device. This count includes the ICRC which is
9621 * not part of the packet held in memory but it is appended
9622 * by the HW.
9623 * This is dependent on the device's receive header entry size.
9624 * HFI allows this to be set per-receive context, but the
9625 * driver presently enforces a global value.
9626 */
9627u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
9628{
9629 /*
9630 * The maximum non-payload (MTU) bytes in LRH.PktLen are
9631 * the Receive Header Entry Size minus the PBC (or RHF) size
9632 * plus one DW for the ICRC appended by HW.
9633 *
9634 * dd->rcd[0].rcvhdrqentsize is in DW.
9635 * We use rcd[0] as all context will have the same value. Also,
9636 * the first kernel context would have been allocated by now so
9637 * we are guaranteed a valid value.
9638 */
9639 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
9640}
9641
9642/*
9643 * Set Send Length
9644 * @ppd - per port data
9645 *
9646 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
9647 * registers compare against LRH.PktLen, so use the max bytes included
9648 * in the LRH.
9649 *
9650 * This routine changes all VL values except VL15, which it maintains at
9651 * the same value.
9652 */
9653static void set_send_length(struct hfi1_pportdata *ppd)
9654{
9655 struct hfi1_devdata *dd = ppd->dd;
6cc6ad2e
HC		 9656 u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
9657 u32 maxvlmtu = dd->vld[15].mtu;
77241056
MM		 9658 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
9659 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
9660 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
9661 int i;
9662
9663 for (i = 0; i < ppd->vls_supported; i++) {
9664 if (dd->vld[i].mtu > maxvlmtu)
9665 maxvlmtu = dd->vld[i].mtu;
9666 if (i <= 3)
9667 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
9668 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
9669 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
9670 else
9671 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
9672 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
9673 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
9674 }
9675 write_csr(dd, SEND_LEN_CHECK0, len1);
9676 write_csr(dd, SEND_LEN_CHECK1, len2);
9677 /* adjust kernel credit return thresholds based on new MTUs */
9678 /* all kernel receive contexts have the same hdrqentsize */
9679 for (i = 0; i < ppd->vls_supported; i++) {
9680 sc_set_cr_threshold(dd->vld[i].sc,
17fb4f29
JJ		 9681 sc_mtu_to_threshold(dd->vld[i].sc,
9682 dd->vld[i].mtu,
9683 dd->rcd[0]->
9684 rcvhdrqentsize));
77241056
MM		 9685 }
9686 sc_set_cr_threshold(dd->vld[15].sc,
17fb4f29
JJ		 9687 sc_mtu_to_threshold(dd->vld[15].sc,
9688 dd->vld[15].mtu,
9689 dd->rcd[0]->rcvhdrqentsize));
77241056
MM		 9690
9691 /* Adjust maximum MTU for the port in DC */
9692 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
9693 (ilog2(maxvlmtu >> 8) + 1);
9694 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
9695 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
9696 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
9697 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
9698 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
9699}
9700
9701static void set_lidlmc(struct hfi1_pportdata *ppd)
9702{
9703 int i;
9704 u64 sreg = 0;
9705 struct hfi1_devdata *dd = ppd->dd;
9706 u32 mask = ~((1U << ppd->lmc) - 1);
9707 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
9708
9709 if (dd->hfi1_snoop.mode_flag)
9710 dd_dev_info(dd, "Set lid/lmc while snooping");
9711
9712 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
9713 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
9714 c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
8638b77f 9715 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
77241056
MM		 9716 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
9717 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
9718 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
9719
9720 /*
9721 * Iterate over all the send contexts and set their SLID check
9722 */
9723 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
9724 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
9725 (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
9726 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
9727
9728 for (i = 0; i < dd->chip_send_contexts; i++) {
9729 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
9730 i, (u32)sreg);
9731 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
9732 }
9733
9734 /* Now we have to do the same thing for the sdma engines */
9735 sdma_update_lmc(dd, mask, ppd->lid);
9736}
9737
9738static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
9739{
9740 unsigned long timeout;
9741 u32 curr_state;
9742
9743 timeout = jiffies + msecs_to_jiffies(msecs);
9744 while (1) {
9745 curr_state = read_physical_state(dd);
9746 if (curr_state == state)
9747 break;
9748 if (time_after(jiffies, timeout)) {
9749 dd_dev_err(dd,
17fb4f29
JJ		 9750 "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
9751 state, curr_state);
77241056
MM		 9752 return -ETIMEDOUT;
9753 }
9754 usleep_range(1950, 2050); /* sleep 2ms-ish */
9755 }
9756
9757 return 0;
9758}
9759
9760/*
9761 * Helper for set_link_state(). Do not call except from that routine.
9762 * Expects ppd->hls_mutex to be held.
9763 *
9764 * @rem_reason value to be sent to the neighbor
9765 *
9766 * LinkDownReasons only set if transition succeeds.
9767 */
9768static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
9769{
9770 struct hfi1_devdata *dd = ppd->dd;
9771 u32 pstate, previous_state;
9772 u32 last_local_state;
9773 u32 last_remote_state;
9774 int ret;
9775 int do_transition;
9776 int do_wait;
9777
9778 previous_state = ppd->host_link_state;
9779 ppd->host_link_state = HLS_GOING_OFFLINE;
9780 pstate = read_physical_state(dd);
9781 if (pstate == PLS_OFFLINE) {
9782 do_transition = 0; /* in right state */
9783 do_wait = 0; /* ...no need to wait */
9784 } else if ((pstate & 0xff) == PLS_OFFLINE) {
9785 do_transition = 0; /* in an offline transient state */
9786 do_wait = 1; /* ...wait for it to settle */
9787 } else {
9788 do_transition = 1; /* need to move to offline */
9789 do_wait = 1; /* ...will need to wait */
9790 }
9791
9792 if (do_transition) {
9793 ret = set_physical_link_state(dd,
17fb4f29 9794 PLS_OFFLINE | (rem_reason << 8));
77241056
MM		 9795
9796 if (ret != HCMD_SUCCESS) {
9797 dd_dev_err(dd,
17fb4f29
JJ		 9798 "Failed to transition to Offline link state, return %d\n",
9799 ret);
77241056
MM		 9800 return -EINVAL;
9801 }
a9c05e35
BM		 9802 if (ppd->offline_disabled_reason ==
9803 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
77241056 9804 ppd->offline_disabled_reason =
a9c05e35 9805 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
77241056
MM		 9806 }
9807
9808 if (do_wait) {
9809 /* it can take a while for the link to go down */
dc060245 9810 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
77241056
MM		 9811 if (ret < 0)
9812 return ret;
9813 }
9814
9815 /* make sure the logical state is also down */
9816 wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
9817
9818 /*
9819 * Now in charge of LCB - must be after the physical state is
9820 * offline.quiet and before host_link_state is changed.
9821 */
9822 set_host_lcb_access(dd);
9823 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9824 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
9825
8ebd4cf1
EH		 9826 if (ppd->port_type == PORT_TYPE_QSFP &&
9827 ppd->qsfp_info.limiting_active &&
9828 qsfp_mod_present(ppd)) {
9829 set_qsfp_tx(ppd, 0);
9830 }
9831
77241056
MM		 9832 /*
9833 * The LNI has a mandatory wait time after the physical state
9834 * moves to Offline.Quiet. The wait time may be different
9835 * depending on how the link went down. The 8051 firmware
9836 * will observe the needed wait time and only move to ready
9837 * when that is completed. The largest of the quiet timeouts
05087f3b
DL		 9838 * is 6s, so wait that long and then at least 0.5s more for
9839 * other transitions, and another 0.5s for a buffer.
77241056 9840 */
05087f3b 9841 ret = wait_fm_ready(dd, 7000);
77241056
MM		 9842 if (ret) {
9843 dd_dev_err(dd,
17fb4f29 9844 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
77241056
MM		 9845 /* state is really offline, so make it so */
9846 ppd->host_link_state = HLS_DN_OFFLINE;
9847 return ret;
9848 }
9849
9850 /*
9851 * The state is now offline and the 8051 is ready to accept host
9852 * requests.
9853 * - change our state
9854 * - notify others if we were previously in a linkup state
9855 */
9856 ppd->host_link_state = HLS_DN_OFFLINE;
9857 if (previous_state & HLS_UP) {
9858 /* went down while link was up */
9859 handle_linkup_change(dd, 0);
9860 } else if (previous_state
9861 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
9862 /* went down while attempting link up */
9863 /* byte 1 of last_*_state is the failure reason */
9864 read_last_local_state(dd, &last_local_state);
9865 read_last_remote_state(dd, &last_remote_state);
9866 dd_dev_err(dd,
17fb4f29
JJ		 9867 "LNI failure last states: local 0x%08x, remote 0x%08x\n",
9868 last_local_state, last_remote_state);
77241056
MM		 9869 }
9870
9871 /* the active link width (downgrade) is 0 on link down */
9872 ppd->link_width_active = 0;
9873 ppd->link_width_downgrade_tx_active = 0;
9874 ppd->link_width_downgrade_rx_active = 0;
9875 ppd->current_egress_rate = 0;
9876 return 0;
9877}
9878
9879/* return the link state name */
9880static const char *link_state_name(u32 state)
9881{
9882 const char *name;
9883 int n = ilog2(state);
9884 static const char * const names[] = {
9885 [__HLS_UP_INIT_BP] = "INIT",
9886 [__HLS_UP_ARMED_BP] = "ARMED",
9887 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
9888 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
9889 [__HLS_DN_POLL_BP] = "POLL",
9890 [__HLS_DN_DISABLE_BP] = "DISABLE",
9891 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
9892 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
9893 [__HLS_GOING_UP_BP] = "GOING_UP",
9894 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
9895 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
9896 };
9897
9898 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
9899 return name ? name : "unknown";
9900}
9901
9902/* return the link state reason name */
9903static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
9904{
9905 if (state == HLS_UP_INIT) {
9906 switch (ppd->linkinit_reason) {
9907 case OPA_LINKINIT_REASON_LINKUP:
9908 return "(LINKUP)";
9909 case OPA_LINKINIT_REASON_FLAPPING:
9910 return "(FLAPPING)";
9911 case OPA_LINKINIT_OUTSIDE_POLICY:
9912 return "(OUTSIDE_POLICY)";
9913 case OPA_LINKINIT_QUARANTINED:
9914 return "(QUARANTINED)";
9915 case OPA_LINKINIT_INSUFIC_CAPABILITY:
9916 return "(INSUFIC_CAPABILITY)";
9917 default:
9918 break;
9919 }
9920 }
9921 return "";
9922}
9923
9924/*
9925 * driver_physical_state - convert the driver's notion of a port's
9926 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
9927 * Return -1 (converted to a u32) to indicate error.
9928 */
9929u32 driver_physical_state(struct hfi1_pportdata *ppd)
9930{
9931 switch (ppd->host_link_state) {
9932 case HLS_UP_INIT:
9933 case HLS_UP_ARMED:
9934 case HLS_UP_ACTIVE:
9935 return IB_PORTPHYSSTATE_LINKUP;
9936 case HLS_DN_POLL:
9937 return IB_PORTPHYSSTATE_POLLING;
9938 case HLS_DN_DISABLE:
9939 return IB_PORTPHYSSTATE_DISABLED;
9940 case HLS_DN_OFFLINE:
9941 return OPA_PORTPHYSSTATE_OFFLINE;
9942 case HLS_VERIFY_CAP:
9943 return IB_PORTPHYSSTATE_POLLING;
9944 case HLS_GOING_UP:
9945 return IB_PORTPHYSSTATE_POLLING;
9946 case HLS_GOING_OFFLINE:
9947 return OPA_PORTPHYSSTATE_OFFLINE;
9948 case HLS_LINK_COOLDOWN:
9949 return OPA_PORTPHYSSTATE_OFFLINE;
9950 case HLS_DN_DOWNDEF:
9951 default:
9952 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
9953 ppd->host_link_state);
9954 return -1;
9955 }
9956}
9957
9958/*
9959 * driver_logical_state - convert the driver's notion of a port's
9960 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
9961 * (converted to a u32) to indicate error.
9962 */
9963u32 driver_logical_state(struct hfi1_pportdata *ppd)
9964{
9965 if (ppd->host_link_state && !(ppd->host_link_state & HLS_UP))
9966 return IB_PORT_DOWN;
9967
9968 switch (ppd->host_link_state & HLS_UP) {
9969 case HLS_UP_INIT:
9970 return IB_PORT_INIT;
9971 case HLS_UP_ARMED:
9972 return IB_PORT_ARMED;
9973 case HLS_UP_ACTIVE:
9974 return IB_PORT_ACTIVE;
9975 default:
9976 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
9977 ppd->host_link_state);
9978 return -1;
9979 }
9980}
9981
9982void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
9983 u8 neigh_reason, u8 rem_reason)
9984{
9985 if (ppd->local_link_down_reason.latest == 0 &&
9986 ppd->neigh_link_down_reason.latest == 0) {
9987 ppd->local_link_down_reason.latest = lcl_reason;
9988 ppd->neigh_link_down_reason.latest = neigh_reason;
9989 ppd->remote_link_down_reason = rem_reason;
9990 }
9991}
9992
9993/*
9994 * Change the physical and/or logical link state.
9995 *
9996 * Do not call this routine while inside an interrupt. It contains
9997 * calls to routines that can take multiple seconds to finish.
9998 *
9999 * Returns 0 on success, -errno on failure.
10000 */
10001int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10002{
10003 struct hfi1_devdata *dd = ppd->dd;
10004 struct ib_event event = {.device = NULL};
10005 int ret1, ret = 0;
10006 int was_up, is_down;
10007 int orig_new_state, poll_bounce;
10008
10009 mutex_lock(&ppd->hls_lock);
10010
10011 orig_new_state = state;
10012 if (state == HLS_DN_DOWNDEF)
10013 state = dd->link_default;
10014
10015 /* interpret poll -> poll as a link bounce */
d0d236ea
JJ		 10016 poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10017 state == HLS_DN_POLL;
77241056
MM		 10018
10019 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
17fb4f29
JJ		 10020 link_state_name(ppd->host_link_state),
10021 link_state_name(orig_new_state),
10022 poll_bounce ? "(bounce) " : "",
10023 link_state_reason_name(ppd, state));
77241056
MM		 10024
10025 was_up = !!(ppd->host_link_state & HLS_UP);
10026
10027 /*
10028 * If we're going to a (HLS_*) link state that implies the logical
10029 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10030 * reset is_sm_config_started to 0.
10031 */
10032 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10033 ppd->is_sm_config_started = 0;
10034
10035 /*
10036 * Do nothing if the states match. Let a poll to poll link bounce
10037 * go through.
10038 */
10039 if (ppd->host_link_state == state && !poll_bounce)
10040 goto done;
10041
10042 switch (state) {
10043 case HLS_UP_INIT:
d0d236ea
JJ		 10044 if (ppd->host_link_state == HLS_DN_POLL &&
10045 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 10046 /*
10047 * Quick link up jumps from polling to here.
10048 *
10049 * Whether in normal or loopback mode, the
10050 * simulator jumps from polling to link up.
10051 * Accept that here.
10052 */
17fb4f29 10053 /* OK */
77241056
MM		 10054 } else if (ppd->host_link_state != HLS_GOING_UP) {
10055 goto unexpected;
10056 }
10057
10058 ppd->host_link_state = HLS_UP_INIT;
10059 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10060 if (ret) {
10061 /* logical state didn't change, stay at going_up */
10062 ppd->host_link_state = HLS_GOING_UP;
10063 dd_dev_err(dd,
17fb4f29
JJ		 10064 "%s: logical state did not change to INIT\n",
10065 __func__);
77241056
MM		 10066 } else {
10067 /* clear old transient LINKINIT_REASON code */
10068 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10069 ppd->linkinit_reason =
10070 OPA_LINKINIT_REASON_LINKUP;
10071
10072 /* enable the port */
10073 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10074
10075 handle_linkup_change(dd, 1);
10076 }
10077 break;
10078 case HLS_UP_ARMED:
10079 if (ppd->host_link_state != HLS_UP_INIT)
10080 goto unexpected;
10081
10082 ppd->host_link_state = HLS_UP_ARMED;
10083 set_logical_state(dd, LSTATE_ARMED);
10084 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10085 if (ret) {
10086 /* logical state didn't change, stay at init */
10087 ppd->host_link_state = HLS_UP_INIT;
10088 dd_dev_err(dd,
17fb4f29
JJ		 10089 "%s: logical state did not change to ARMED\n",
10090 __func__);
77241056
MM		 10091 }
10092 /*
10093 * The simulator does not currently implement SMA messages,
10094 * so neighbor_normal is not set. Set it here when we first
10095 * move to Armed.
10096 */
10097 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10098 ppd->neighbor_normal = 1;
10099 break;
10100 case HLS_UP_ACTIVE:
10101 if (ppd->host_link_state != HLS_UP_ARMED)
10102 goto unexpected;
10103
10104 ppd->host_link_state = HLS_UP_ACTIVE;
10105 set_logical_state(dd, LSTATE_ACTIVE);
10106 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10107 if (ret) {
10108 /* logical state didn't change, stay at armed */
10109 ppd->host_link_state = HLS_UP_ARMED;
10110 dd_dev_err(dd,
17fb4f29
JJ		 10111 "%s: logical state did not change to ACTIVE\n",
10112 __func__);
77241056 10113 } else {
77241056
MM		 10114 /* tell all engines to go running */
10115 sdma_all_running(dd);
10116
10117 /* Signal the IB layer that the port has went active */
ec3f2c12 10118 event.device = &dd->verbs_dev.rdi.ibdev;
77241056
MM		 10119 event.element.port_num = ppd->port;
10120 event.event = IB_EVENT_PORT_ACTIVE;
10121 }
10122 break;
10123 case HLS_DN_POLL:
10124 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10125 ppd->host_link_state == HLS_DN_OFFLINE) &&
10126 dd->dc_shutdown)
10127 dc_start(dd);
10128 /* Hand LED control to the DC */
10129 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10130
10131 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10132 u8 tmp = ppd->link_enabled;
10133
10134 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10135 if (ret) {
10136 ppd->link_enabled = tmp;
10137 break;
10138 }
10139 ppd->remote_link_down_reason = 0;
10140
10141 if (ppd->driver_link_ready)
10142 ppd->link_enabled = 1;
10143 }
10144
fb9036dd 10145 set_all_slowpath(ppd->dd);
77241056
MM		 10146 ret = set_local_link_attributes(ppd);
10147 if (ret)
10148 break;
10149
10150 ppd->port_error_action = 0;
10151 ppd->host_link_state = HLS_DN_POLL;
10152
10153 if (quick_linkup) {
10154 /* quick linkup does not go into polling */
10155 ret = do_quick_linkup(dd);
10156 } else {
10157 ret1 = set_physical_link_state(dd, PLS_POLLING);
10158 if (ret1 != HCMD_SUCCESS) {
10159 dd_dev_err(dd,
17fb4f29
JJ		 10160 "Failed to transition to Polling link state, return 0x%x\n",
10161 ret1);
77241056
MM		 10162 ret = -EINVAL;
10163 }
10164 }
a9c05e35
BM		 10165 ppd->offline_disabled_reason =
10166 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
77241056
MM		 10167 /*
10168 * If an error occurred above, go back to offline. The
10169 * caller may reschedule another attempt.
10170 */
10171 if (ret)
10172 goto_offline(ppd, 0);
10173 break;
10174 case HLS_DN_DISABLE:
10175 /* link is disabled */
10176 ppd->link_enabled = 0;
10177
10178 /* allow any state to transition to disabled */
10179
10180 /* must transition to offline first */
10181 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10182 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10183 if (ret)
10184 break;
10185 ppd->remote_link_down_reason = 0;
10186 }
10187
10188 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10189 if (ret1 != HCMD_SUCCESS) {
10190 dd_dev_err(dd,
17fb4f29
JJ		 10191 "Failed to transition to Disabled link state, return 0x%x\n",
10192 ret1);
77241056
MM		 10193 ret = -EINVAL;
10194 break;
10195 }
10196 ppd->host_link_state = HLS_DN_DISABLE;
10197 dc_shutdown(dd);
10198 break;
10199 case HLS_DN_OFFLINE:
10200 if (ppd->host_link_state == HLS_DN_DISABLE)
10201 dc_start(dd);
10202
10203 /* allow any state to transition to offline */
10204 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10205 if (!ret)
10206 ppd->remote_link_down_reason = 0;
10207 break;
10208 case HLS_VERIFY_CAP:
10209 if (ppd->host_link_state != HLS_DN_POLL)
10210 goto unexpected;
10211 ppd->host_link_state = HLS_VERIFY_CAP;
10212 break;
10213 case HLS_GOING_UP:
10214 if (ppd->host_link_state != HLS_VERIFY_CAP)
10215 goto unexpected;
10216
10217 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10218 if (ret1 != HCMD_SUCCESS) {
10219 dd_dev_err(dd,
17fb4f29
JJ		 10220 "Failed to transition to link up state, return 0x%x\n",
10221 ret1);
77241056
MM		 10222 ret = -EINVAL;
10223 break;
10224 }
10225 ppd->host_link_state = HLS_GOING_UP;
10226 break;
10227
10228 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
10229 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
10230 default:
10231 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
17fb4f29 10232 __func__, state);
77241056
MM		 10233 ret = -EINVAL;
10234 break;
10235 }
10236
10237 is_down = !!(ppd->host_link_state & (HLS_DN_POLL |
10238 HLS_DN_DISABLE | HLS_DN_OFFLINE));
10239
10240 if (was_up && is_down && ppd->local_link_down_reason.sma == 0 &&
10241 ppd->neigh_link_down_reason.sma == 0) {
10242 ppd->local_link_down_reason.sma =
10243 ppd->local_link_down_reason.latest;
10244 ppd->neigh_link_down_reason.sma =
10245 ppd->neigh_link_down_reason.latest;
10246 }
10247
10248 goto done;
10249
10250unexpected:
10251 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
17fb4f29
JJ		 10252 __func__, link_state_name(ppd->host_link_state),
10253 link_state_name(state));
77241056
MM		 10254 ret = -EINVAL;
10255
10256done:
10257 mutex_unlock(&ppd->hls_lock);
10258
10259 if (event.device)
10260 ib_dispatch_event(&event);
10261
10262 return ret;
10263}
10264
10265int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10266{
10267 u64 reg;
10268 int ret = 0;
10269
10270 switch (which) {
10271 case HFI1_IB_CFG_LIDLMC:
10272 set_lidlmc(ppd);
10273 break;
10274 case HFI1_IB_CFG_VL_HIGH_LIMIT:
10275 /*
10276 * The VL Arbitrator high limit is sent in units of 4k
10277 * bytes, while HFI stores it in units of 64 bytes.
10278 */
8638b77f 10279 val *= 4096 / 64;
77241056
MM		 10280 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10281 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10282 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10283 break;
10284 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10285 /* HFI only supports POLL as the default link down state */
10286 if (val != HLS_DN_POLL)
10287 ret = -EINVAL;
10288 break;
10289 case HFI1_IB_CFG_OP_VLS:
10290 if (ppd->vls_operational != val) {
10291 ppd->vls_operational = val;
10292 if (!ppd->port)
10293 ret = -EINVAL;
77241056
MM		 10294 }
10295 break;
10296 /*
10297 * For link width, link width downgrade, and speed enable, always AND
10298 * the setting with what is actually supported. This has two benefits.
10299 * First, enabled can't have unsupported values, no matter what the
10300 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
10301 * "fill in with your supported value" have all the bits in the
10302 * field set, so simply ANDing with supported has the desired result.
10303 */
10304 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
10305 ppd->link_width_enabled = val & ppd->link_width_supported;
10306 break;
10307 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
10308 ppd->link_width_downgrade_enabled =
10309 val & ppd->link_width_downgrade_supported;
10310 break;
10311 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10312 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10313 break;
10314 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10315 /*
10316 * HFI does not follow IB specs, save this value
10317 * so we can report it, if asked.
10318 */
10319 ppd->overrun_threshold = val;
10320 break;
10321 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10322 /*
10323 * HFI does not follow IB specs, save this value
10324 * so we can report it, if asked.
10325 */
10326 ppd->phy_error_threshold = val;
10327 break;
10328
10329 case HFI1_IB_CFG_MTU:
10330 set_send_length(ppd);
10331 break;
10332
10333 case HFI1_IB_CFG_PKEYS:
10334 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10335 set_partition_keys(ppd);
10336 break;
10337
10338 default:
10339 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10340 dd_dev_info(ppd->dd,
17fb4f29
JJ		 10341 "%s: which %s, val 0x%x: not implemented\n",
10342 __func__, ib_cfg_name(which), val);
77241056
MM		 10343 break;
10344 }
10345 return ret;
10346}
10347
10348/* begin functions related to vl arbitration table caching */
10349static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
10350{
10351 int i;
10352
10353 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10354 VL_ARB_LOW_PRIO_TABLE_SIZE);
10355 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10356 VL_ARB_HIGH_PRIO_TABLE_SIZE);
10357
10358 /*
10359 * Note that we always return values directly from the
10360 * 'vl_arb_cache' (and do no CSR reads) in response to a
10361 * 'Get(VLArbTable)'. This is obviously correct after a
10362 * 'Set(VLArbTable)', since the cache will then be up to
10363 * date. But it's also correct prior to any 'Set(VLArbTable)'
10364 * since then both the cache, and the relevant h/w registers
10365 * will be zeroed.
10366 */
10367
10368 for (i = 0; i < MAX_PRIO_TABLE; i++)
10369 spin_lock_init(&ppd->vl_arb_cache[i].lock);
10370}
10371
10372/*
10373 * vl_arb_lock_cache
10374 *
10375 * All other vl_arb_* functions should be called only after locking
10376 * the cache.
10377 */
10378static inline struct vl_arb_cache *
10379vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
10380{
10381 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
10382 return NULL;
10383 spin_lock(&ppd->vl_arb_cache[idx].lock);
10384 return &ppd->vl_arb_cache[idx];
10385}
10386
10387static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
10388{
10389 spin_unlock(&ppd->vl_arb_cache[idx].lock);
10390}
10391
10392static void vl_arb_get_cache(struct vl_arb_cache *cache,
10393 struct ib_vl_weight_elem *vl)
10394{
10395 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
10396}
10397
10398static void vl_arb_set_cache(struct vl_arb_cache *cache,
10399 struct ib_vl_weight_elem *vl)
10400{
10401 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10402}
10403
10404static int vl_arb_match_cache(struct vl_arb_cache *cache,
10405 struct ib_vl_weight_elem *vl)
10406{
10407 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10408}
f4d507cd 10409
77241056
MM		 10410/* end functions related to vl arbitration table caching */
10411
10412static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
10413 u32 size, struct ib_vl_weight_elem *vl)
10414{
10415 struct hfi1_devdata *dd = ppd->dd;
10416 u64 reg;
10417 unsigned int i, is_up = 0;
10418 int drain, ret = 0;
10419
10420 mutex_lock(&ppd->hls_lock);
10421
10422 if (ppd->host_link_state & HLS_UP)
10423 is_up = 1;
10424
10425 drain = !is_ax(dd) && is_up;
10426
10427 if (drain)
10428 /*
10429 * Before adjusting VL arbitration weights, empty per-VL
10430 * FIFOs, otherwise a packet whose VL weight is being
10431 * set to 0 could get stuck in a FIFO with no chance to
10432 * egress.
10433 */
10434 ret = stop_drain_data_vls(dd);
10435
10436 if (ret) {
10437 dd_dev_err(
10438 dd,
10439 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
10440 __func__);
10441 goto err;
10442 }
10443
10444 for (i = 0; i < size; i++, vl++) {
10445 /*
10446 * NOTE: The low priority shift and mask are used here, but
10447 * they are the same for both the low and high registers.
10448 */
10449 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
10450 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
10451 | (((u64)vl->weight
10452 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
10453 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
10454 write_csr(dd, target + (i * 8), reg);
10455 }
10456 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
10457
10458 if (drain)
10459 open_fill_data_vls(dd); /* reopen all VLs */
10460
10461err:
10462 mutex_unlock(&ppd->hls_lock);
10463
10464 return ret;
10465}
10466
10467/*
10468 * Read one credit merge VL register.
10469 */
10470static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
10471 struct vl_limit *vll)
10472{
10473 u64 reg = read_csr(dd, csr);
10474
10475 vll->dedicated = cpu_to_be16(
10476 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
10477 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
10478 vll->shared = cpu_to_be16(
10479 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
10480 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
10481}
10482
10483/*
10484 * Read the current credit merge limits.
10485 */
10486static int get_buffer_control(struct hfi1_devdata *dd,
10487 struct buffer_control *bc, u16 *overall_limit)
10488{
10489 u64 reg;
10490 int i;
10491
10492 /* not all entries are filled in */
10493 memset(bc, 0, sizeof(*bc));
10494
10495 /* OPA and HFI have a 1-1 mapping */
10496 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 10497 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
77241056
MM		 10498
10499 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
10500 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
10501
10502 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10503 bc->overall_shared_limit = cpu_to_be16(
10504 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
10505 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
10506 if (overall_limit)
10507 *overall_limit = (reg
10508 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
10509 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
10510 return sizeof(struct buffer_control);
10511}
10512
10513static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10514{
10515 u64 reg;
10516 int i;
10517
10518 /* each register contains 16 SC->VLnt mappings, 4 bits each */
10519 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
10520 for (i = 0; i < sizeof(u64); i++) {
10521 u8 byte = *(((u8 *)&reg) + i);
10522
10523 dp->vlnt[2 * i] = byte & 0xf;
10524 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
10525 }
10526
10527 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
10528 for (i = 0; i < sizeof(u64); i++) {
10529 u8 byte = *(((u8 *)&reg) + i);
10530
10531 dp->vlnt[16 + (2 * i)] = byte & 0xf;
10532 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
10533 }
10534 return sizeof(struct sc2vlnt);
10535}
10536
10537static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
10538 struct ib_vl_weight_elem *vl)
10539{
10540 unsigned int i;
10541
10542 for (i = 0; i < nelems; i++, vl++) {
10543 vl->vl = 0xf;
10544 vl->weight = 0;
10545 }
10546}
10547
10548static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10549{
10550 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
17fb4f29
JJ		 10551 DC_SC_VL_VAL(15_0,
10552 0, dp->vlnt[0] & 0xf,
10553 1, dp->vlnt[1] & 0xf,
10554 2, dp->vlnt[2] & 0xf,
10555 3, dp->vlnt[3] & 0xf,
10556 4, dp->vlnt[4] & 0xf,
10557 5, dp->vlnt[5] & 0xf,
10558 6, dp->vlnt[6] & 0xf,
10559 7, dp->vlnt[7] & 0xf,
10560 8, dp->vlnt[8] & 0xf,
10561 9, dp->vlnt[9] & 0xf,
10562 10, dp->vlnt[10] & 0xf,
10563 11, dp->vlnt[11] & 0xf,
10564 12, dp->vlnt[12] & 0xf,
10565 13, dp->vlnt[13] & 0xf,
10566 14, dp->vlnt[14] & 0xf,
10567 15, dp->vlnt[15] & 0xf));
77241056 10568 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
17fb4f29
JJ		 10569 DC_SC_VL_VAL(31_16,
10570 16, dp->vlnt[16] & 0xf,
10571 17, dp->vlnt[17] & 0xf,
10572 18, dp->vlnt[18] & 0xf,
10573 19, dp->vlnt[19] & 0xf,
10574 20, dp->vlnt[20] & 0xf,
10575 21, dp->vlnt[21] & 0xf,
10576 22, dp->vlnt[22] & 0xf,
10577 23, dp->vlnt[23] & 0xf,
10578 24, dp->vlnt[24] & 0xf,
10579 25, dp->vlnt[25] & 0xf,
10580 26, dp->vlnt[26] & 0xf,
10581 27, dp->vlnt[27] & 0xf,
10582 28, dp->vlnt[28] & 0xf,
10583 29, dp->vlnt[29] & 0xf,
10584 30, dp->vlnt[30] & 0xf,
10585 31, dp->vlnt[31] & 0xf));
77241056
MM		 10586}
10587
10588static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
10589 u16 limit)
10590{
10591 if (limit != 0)
10592 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
17fb4f29 10593 what, (int)limit, idx);
77241056
MM		 10594}
10595
10596/* change only the shared limit portion of SendCmGLobalCredit */
10597static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
10598{
10599 u64 reg;
10600
10601 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10602 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
10603 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
10604 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10605}
10606
10607/* change only the total credit limit portion of SendCmGLobalCredit */
10608static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
10609{
10610 u64 reg;
10611
10612 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10613 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
10614 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
10615 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10616}
10617
10618/* set the given per-VL shared limit */
10619static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
10620{
10621 u64 reg;
10622 u32 addr;
10623
10624 if (vl < TXE_NUM_DATA_VL)
10625 addr = SEND_CM_CREDIT_VL + (8 * vl);
10626 else
10627 addr = SEND_CM_CREDIT_VL15;
10628
10629 reg = read_csr(dd, addr);
10630 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
10631 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
10632 write_csr(dd, addr, reg);
10633}
10634
10635/* set the given per-VL dedicated limit */
10636static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
10637{
10638 u64 reg;
10639 u32 addr;
10640
10641 if (vl < TXE_NUM_DATA_VL)
10642 addr = SEND_CM_CREDIT_VL + (8 * vl);
10643 else
10644 addr = SEND_CM_CREDIT_VL15;
10645
10646 reg = read_csr(dd, addr);
10647 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
10648 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
10649 write_csr(dd, addr, reg);
10650}
10651
10652/* spin until the given per-VL status mask bits clear */
10653static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
10654 const char *which)
10655{
10656 unsigned long timeout;
10657 u64 reg;
10658
10659 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
10660 while (1) {
10661 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
10662
10663 if (reg == 0)
10664 return; /* success */
10665 if (time_after(jiffies, timeout))
10666 break; /* timed out */
10667 udelay(1);
10668 }
10669
10670 dd_dev_err(dd,
17fb4f29
JJ		 10671 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
10672 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
77241056
MM		 10673 /*
10674 * If this occurs, it is likely there was a credit loss on the link.
10675 * The only recovery from that is a link bounce.
10676 */
10677 dd_dev_err(dd,
17fb4f29 10678 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
77241056
MM		 10679}
10680
10681/*
10682 * The number of credits on the VLs may be changed while everything
10683 * is "live", but the following algorithm must be followed due to
10684 * how the hardware is actually implemented. In particular,
10685 * Return_Credit_Status[] is the only correct status check.
10686 *
10687 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
10688 * set Global_Shared_Credit_Limit = 0
10689 * use_all_vl = 1
10690 * mask0 = all VLs that are changing either dedicated or shared limits
10691 * set Shared_Limit[mask0] = 0
10692 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
10693 * if (changing any dedicated limit)
10694 * mask1 = all VLs that are lowering dedicated limits
10695 * lower Dedicated_Limit[mask1]
10696 * spin until Return_Credit_Status[mask1] == 0
10697 * raise Dedicated_Limits
10698 * raise Shared_Limits
10699 * raise Global_Shared_Credit_Limit
10700 *
10701 * lower = if the new limit is lower, set the limit to the new value
10702 * raise = if the new limit is higher than the current value (may be changed
10703 * earlier in the algorithm), set the new limit to the new value
10704 */
8a4d3444
MM		 10705int set_buffer_control(struct hfi1_pportdata *ppd,
10706 struct buffer_control *new_bc)
77241056 10707{
8a4d3444 10708 struct hfi1_devdata *dd = ppd->dd;
77241056
MM		 10709 u64 changing_mask, ld_mask, stat_mask;
10710 int change_count;
10711 int i, use_all_mask;
10712 int this_shared_changing;
8a4d3444 10713 int vl_count = 0, ret;
77241056
MM		 10714 /*
10715 * A0: add the variable any_shared_limit_changing below and in the
10716 * algorithm above. If removing A0 support, it can be removed.
10717 */
10718 int any_shared_limit_changing;
10719 struct buffer_control cur_bc;
10720 u8 changing[OPA_MAX_VLS];
10721 u8 lowering_dedicated[OPA_MAX_VLS];
10722 u16 cur_total;
10723 u32 new_total = 0;
10724 const u64 all_mask =
10725 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
10726 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
10727 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
10728 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
10729 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
10730 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
10731 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
10732 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
10733 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
10734
10735#define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
10736#define NUM_USABLE_VLS 16 /* look at VL15 and less */
10737
77241056
MM		 10738 /* find the new total credits, do sanity check on unused VLs */
10739 for (i = 0; i < OPA_MAX_VLS; i++) {
10740 if (valid_vl(i)) {
10741 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
10742 continue;
10743 }
10744 nonzero_msg(dd, i, "dedicated",
17fb4f29 10745 be16_to_cpu(new_bc->vl[i].dedicated));
77241056 10746 nonzero_msg(dd, i, "shared",
17fb4f29 10747 be16_to_cpu(new_bc->vl[i].shared));
77241056
MM		 10748 new_bc->vl[i].dedicated = 0;
10749 new_bc->vl[i].shared = 0;
10750 }
10751 new_total += be16_to_cpu(new_bc->overall_shared_limit);
bff14bb6 10752
77241056
MM		 10753 /* fetch the current values */
10754 get_buffer_control(dd, &cur_bc, &cur_total);
10755
10756 /*
10757 * Create the masks we will use.
10758 */
10759 memset(changing, 0, sizeof(changing));
10760 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
4d114fdd
JJ		 10761 /*
10762 * NOTE: Assumes that the individual VL bits are adjacent and in
10763 * increasing order
10764 */
77241056
MM		 10765 stat_mask =
10766 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
10767 changing_mask = 0;
10768 ld_mask = 0;
10769 change_count = 0;
10770 any_shared_limit_changing = 0;
10771 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
10772 if (!valid_vl(i))
10773 continue;
10774 this_shared_changing = new_bc->vl[i].shared
10775 != cur_bc.vl[i].shared;
10776 if (this_shared_changing)
10777 any_shared_limit_changing = 1;
d0d236ea
JJ		 10778 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
10779 this_shared_changing) {
77241056
MM		 10780 changing[i] = 1;
10781 changing_mask |= stat_mask;
10782 change_count++;
10783 }
10784 if (be16_to_cpu(new_bc->vl[i].dedicated) <
10785 be16_to_cpu(cur_bc.vl[i].dedicated)) {
10786 lowering_dedicated[i] = 1;
10787 ld_mask |= stat_mask;
10788 }
10789 }
10790
10791 /* bracket the credit change with a total adjustment */
10792 if (new_total > cur_total)
10793 set_global_limit(dd, new_total);
10794
10795 /*
10796 * Start the credit change algorithm.
10797 */
10798 use_all_mask = 0;
10799 if ((be16_to_cpu(new_bc->overall_shared_limit) <
995deafa
MM		 10800 be16_to_cpu(cur_bc.overall_shared_limit)) ||
10801 (is_ax(dd) && any_shared_limit_changing)) {
77241056
MM		 10802 set_global_shared(dd, 0);
10803 cur_bc.overall_shared_limit = 0;
10804 use_all_mask = 1;
10805 }
10806
10807 for (i = 0; i < NUM_USABLE_VLS; i++) {
10808 if (!valid_vl(i))
10809 continue;
10810
10811 if (changing[i]) {
10812 set_vl_shared(dd, i, 0);
10813 cur_bc.vl[i].shared = 0;
10814 }
10815 }
10816
10817 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
17fb4f29 10818 "shared");
77241056
MM		 10819
10820 if (change_count > 0) {
10821 for (i = 0; i < NUM_USABLE_VLS; i++) {
10822 if (!valid_vl(i))
10823 continue;
10824
10825 if (lowering_dedicated[i]) {
10826 set_vl_dedicated(dd, i,
17fb4f29
JJ		 10827 be16_to_cpu(new_bc->
10828 vl[i].dedicated));
77241056
MM		 10829 cur_bc.vl[i].dedicated =
10830 new_bc->vl[i].dedicated;
10831 }
10832 }
10833
10834 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
10835
10836 /* now raise all dedicated that are going up */
10837 for (i = 0; i < NUM_USABLE_VLS; i++) {
10838 if (!valid_vl(i))
10839 continue;
10840
10841 if (be16_to_cpu(new_bc->vl[i].dedicated) >
10842 be16_to_cpu(cur_bc.vl[i].dedicated))
10843 set_vl_dedicated(dd, i,
17fb4f29
JJ		 10844 be16_to_cpu(new_bc->
10845 vl[i].dedicated));
77241056
MM		 10846 }
10847 }
10848
10849 /* next raise all shared that are going up */
10850 for (i = 0; i < NUM_USABLE_VLS; i++) {
10851 if (!valid_vl(i))
10852 continue;
10853
10854 if (be16_to_cpu(new_bc->vl[i].shared) >
10855 be16_to_cpu(cur_bc.vl[i].shared))
10856 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
10857 }
10858
10859 /* finally raise the global shared */
10860 if (be16_to_cpu(new_bc->overall_shared_limit) >
17fb4f29 10861 be16_to_cpu(cur_bc.overall_shared_limit))
77241056 10862 set_global_shared(dd,
17fb4f29 10863 be16_to_cpu(new_bc->overall_shared_limit));
77241056
MM		 10864
10865 /* bracket the credit change with a total adjustment */
10866 if (new_total < cur_total)
10867 set_global_limit(dd, new_total);
8a4d3444
MM		 10868
10869 /*
10870 * Determine the actual number of operational VLS using the number of
10871 * dedicated and shared credits for each VL.
10872 */
10873 if (change_count > 0) {
10874 for (i = 0; i < TXE_NUM_DATA_VL; i++)
10875 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
10876 be16_to_cpu(new_bc->vl[i].shared) > 0)
10877 vl_count++;
10878 ppd->actual_vls_operational = vl_count;
10879 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
10880 ppd->actual_vls_operational :
10881 ppd->vls_operational,
10882 NULL);
10883 if (ret == 0)
10884 ret = pio_map_init(dd, ppd->port - 1, vl_count ?
10885 ppd->actual_vls_operational :
10886 ppd->vls_operational, NULL);
10887 if (ret)
10888 return ret;
10889 }
77241056
MM		 10890 return 0;
10891}
10892
10893/*
10894 * Read the given fabric manager table. Return the size of the
10895 * table (in bytes) on success, and a negative error code on
10896 * failure.
10897 */
10898int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
10899
10900{
10901 int size;
10902 struct vl_arb_cache *vlc;
10903
10904 switch (which) {
10905 case FM_TBL_VL_HIGH_ARB:
10906 size = 256;
10907 /*
10908 * OPA specifies 128 elements (of 2 bytes each), though
10909 * HFI supports only 16 elements in h/w.
10910 */
10911 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
10912 vl_arb_get_cache(vlc, t);
10913 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
10914 break;
10915 case FM_TBL_VL_LOW_ARB:
10916 size = 256;
10917 /*
10918 * OPA specifies 128 elements (of 2 bytes each), though
10919 * HFI supports only 16 elements in h/w.
10920 */
10921 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
10922 vl_arb_get_cache(vlc, t);
10923 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
10924 break;
10925 case FM_TBL_BUFFER_CONTROL:
10926 size = get_buffer_control(ppd->dd, t, NULL);
10927 break;
10928 case FM_TBL_SC2VLNT:
10929 size = get_sc2vlnt(ppd->dd, t);
10930 break;
10931 case FM_TBL_VL_PREEMPT_ELEMS:
10932 size = 256;
10933 /* OPA specifies 128 elements, of 2 bytes each */
10934 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
10935 break;
10936 case FM_TBL_VL_PREEMPT_MATRIX:
10937 size = 256;
10938 /*
10939 * OPA specifies that this is the same size as the VL
10940 * arbitration tables (i.e., 256 bytes).
10941 */
10942 break;
10943 default:
10944 return -EINVAL;
10945 }
10946 return size;
10947}
10948
10949/*
10950 * Write the given fabric manager table.
10951 */
10952int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
10953{
10954 int ret = 0;
10955 struct vl_arb_cache *vlc;
10956
10957 switch (which) {
10958 case FM_TBL_VL_HIGH_ARB:
10959 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
10960 if (vl_arb_match_cache(vlc, t)) {
10961 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
10962 break;
10963 }
10964 vl_arb_set_cache(vlc, t);
10965 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
10966 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
10967 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
10968 break;
10969 case FM_TBL_VL_LOW_ARB:
10970 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
10971 if (vl_arb_match_cache(vlc, t)) {
10972 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
10973 break;
10974 }
10975 vl_arb_set_cache(vlc, t);
10976 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
10977 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
10978 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
10979 break;
10980 case FM_TBL_BUFFER_CONTROL:
8a4d3444 10981 ret = set_buffer_control(ppd, t);
77241056
MM		 10982 break;
10983 case FM_TBL_SC2VLNT:
10984 set_sc2vlnt(ppd->dd, t);
10985 break;
10986 default:
10987 ret = -EINVAL;
10988 }
10989 return ret;
10990}
10991
10992/*
10993 * Disable all data VLs.
10994 *
10995 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
10996 */
10997static int disable_data_vls(struct hfi1_devdata *dd)
10998{
995deafa 10999 if (is_ax(dd))
77241056
MM		 11000 return 1;
11001
11002 pio_send_control(dd, PSC_DATA_VL_DISABLE);
11003
11004 return 0;
11005}
11006
11007/*
11008 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11009 * Just re-enables all data VLs (the "fill" part happens
11010 * automatically - the name was chosen for symmetry with
11011 * stop_drain_data_vls()).
11012 *
11013 * Return 0 if successful, non-zero if the VLs cannot be enabled.
11014 */
11015int open_fill_data_vls(struct hfi1_devdata *dd)
11016{
995deafa 11017 if (is_ax(dd))
77241056
MM		 11018 return 1;
11019
11020 pio_send_control(dd, PSC_DATA_VL_ENABLE);
11021
11022 return 0;
11023}
11024
11025/*
11026 * drain_data_vls() - assumes that disable_data_vls() has been called,
11027 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11028 * engines to drop to 0.
11029 */
11030static void drain_data_vls(struct hfi1_devdata *dd)
11031{
11032 sc_wait(dd);
11033 sdma_wait(dd);
11034 pause_for_credit_return(dd);
11035}
11036
11037/*
11038 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11039 *
11040 * Use open_fill_data_vls() to resume using data VLs. This pair is
11041 * meant to be used like this:
11042 *
11043 * stop_drain_data_vls(dd);
11044 * // do things with per-VL resources
11045 * open_fill_data_vls(dd);
11046 */
11047int stop_drain_data_vls(struct hfi1_devdata *dd)
11048{
11049 int ret;
11050
11051 ret = disable_data_vls(dd);
11052 if (ret == 0)
11053 drain_data_vls(dd);
11054
11055 return ret;
11056}
11057
11058/*
11059 * Convert a nanosecond time to a cclock count. No matter how slow
11060 * the cclock, a non-zero ns will always have a non-zero result.
11061 */
11062u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11063{
11064 u32 cclocks;
11065
11066 if (dd->icode == ICODE_FPGA_EMULATION)
11067 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11068 else /* simulation pretends to be ASIC */
11069 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11070 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
11071 cclocks = 1;
11072 return cclocks;
11073}
11074
11075/*
11076 * Convert a cclock count to nanoseconds. Not matter how slow
11077 * the cclock, a non-zero cclocks will always have a non-zero result.
11078 */
11079u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11080{
11081 u32 ns;
11082
11083 if (dd->icode == ICODE_FPGA_EMULATION)
11084 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11085 else /* simulation pretends to be ASIC */
11086 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11087 if (cclocks && !ns)
11088 ns = 1;
11089 return ns;
11090}
11091
11092/*
11093 * Dynamically adjust the receive interrupt timeout for a context based on
11094 * incoming packet rate.
11095 *
11096 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11097 */
11098static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11099{
11100 struct hfi1_devdata *dd = rcd->dd;
11101 u32 timeout = rcd->rcvavail_timeout;
11102
11103 /*
11104 * This algorithm doubles or halves the timeout depending on whether
11105 * the number of packets received in this interrupt were less than or
11106 * greater equal the interrupt count.
11107 *
11108 * The calculations below do not allow a steady state to be achieved.
11109 * Only at the endpoints it is possible to have an unchanging
11110 * timeout.
11111 */
11112 if (npkts < rcv_intr_count) {
11113 /*
11114 * Not enough packets arrived before the timeout, adjust
11115 * timeout downward.
11116 */
11117 if (timeout < 2) /* already at minimum? */
11118 return;
11119 timeout >>= 1;
11120 } else {
11121 /*
11122 * More than enough packets arrived before the timeout, adjust
11123 * timeout upward.
11124 */
11125 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11126 return;
11127 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11128 }
11129
11130 rcd->rcvavail_timeout = timeout;
4d114fdd
JJ		 11131 /*
11132 * timeout cannot be larger than rcv_intr_timeout_csr which has already
11133 * been verified to be in range
11134 */
77241056 11135 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
17fb4f29
JJ		 11136 (u64)timeout <<
11137 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
77241056
MM		 11138}
11139
11140void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11141 u32 intr_adjust, u32 npkts)
11142{
11143 struct hfi1_devdata *dd = rcd->dd;
11144 u64 reg;
11145 u32 ctxt = rcd->ctxt;
11146
11147 /*
11148 * Need to write timeout register before updating RcvHdrHead to ensure
11149 * that a new value is used when the HW decides to restart counting.
11150 */
11151 if (intr_adjust)
11152 adjust_rcv_timeout(rcd, npkts);
11153 if (updegr) {
11154 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11155 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11156 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11157 }
11158 mmiowb();
11159 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11160 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11161 << RCV_HDR_HEAD_HEAD_SHIFT);
11162 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11163 mmiowb();
11164}
11165
11166u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11167{
11168 u32 head, tail;
11169
11170 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11171 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11172
11173 if (rcd->rcvhdrtail_kvaddr)
11174 tail = get_rcvhdrtail(rcd);
11175 else
11176 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11177
11178 return head == tail;
11179}
11180
11181/*
11182 * Context Control and Receive Array encoding for buffer size:
11183 * 0x0 invalid
11184 * 0x1 4 KB
11185 * 0x2 8 KB
11186 * 0x3 16 KB
11187 * 0x4 32 KB
11188 * 0x5 64 KB
11189 * 0x6 128 KB
11190 * 0x7 256 KB
11191 * 0x8 512 KB (Receive Array only)
11192 * 0x9 1 MB (Receive Array only)
11193 * 0xa 2 MB (Receive Array only)
11194 *
11195 * 0xB-0xF - reserved (Receive Array only)
11196 *
11197 *
11198 * This routine assumes that the value has already been sanity checked.
11199 */
11200static u32 encoded_size(u32 size)
11201{
11202 switch (size) {
8638b77f
JJ		 11203 case 4 * 1024: return 0x1;
11204 case 8 * 1024: return 0x2;
11205 case 16 * 1024: return 0x3;
11206 case 32 * 1024: return 0x4;
11207 case 64 * 1024: return 0x5;
11208 case 128 * 1024: return 0x6;
11209 case 256 * 1024: return 0x7;
11210 case 512 * 1024: return 0x8;
11211 case 1 * 1024 * 1024: return 0x9;
11212 case 2 * 1024 * 1024: return 0xa;
77241056
MM		 11213 }
11214 return 0x1; /* if invalid, go with the minimum size */
11215}
11216
11217void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
11218{
11219 struct hfi1_ctxtdata *rcd;
11220 u64 rcvctrl, reg;
11221 int did_enable = 0;
11222
11223 rcd = dd->rcd[ctxt];
11224 if (!rcd)
11225 return;
11226
11227 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11228
11229 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11230 /* if the context already enabled, don't do the extra steps */
d0d236ea
JJ		 11231 if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11232 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
77241056
MM		 11233 /* reset the tail and hdr addresses, and sequence count */
11234 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11235 rcd->rcvhdrq_phys);
11236 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
11237 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11238 rcd->rcvhdrqtailaddr_phys);
11239 rcd->seq_cnt = 1;
11240
11241 /* reset the cached receive header queue head value */
11242 rcd->head = 0;
11243
11244 /*
11245 * Zero the receive header queue so we don't get false
11246 * positives when checking the sequence number. The
11247 * sequence numbers could land exactly on the same spot.
11248 * E.g. a rcd restart before the receive header wrapped.
11249 */
11250 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
11251
11252 /* starting timeout */
11253 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11254
11255 /* enable the context */
11256 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11257
11258 /* clean the egr buffer size first */
11259 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11260 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11261 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11262 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11263
11264 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
11265 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11266 did_enable = 1;
11267
11268 /* zero RcvEgrIndexHead */
11269 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11270
11271 /* set eager count and base index */
11272 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11273 & RCV_EGR_CTRL_EGR_CNT_MASK)
11274 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11275 (((rcd->eager_base >> RCV_SHIFT)
11276 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11277 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11278 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11279
11280 /*
11281 * Set TID (expected) count and base index.
11282 * rcd->expected_count is set to individual RcvArray entries,
11283 * not pairs, and the CSR takes a pair-count in groups of
11284 * four, so divide by 8.
11285 */
11286 reg = (((rcd->expected_count >> RCV_SHIFT)
11287 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11288 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11289 (((rcd->expected_base >> RCV_SHIFT)
11290 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11291 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11292 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
82c2611d
NV		 11293 if (ctxt == HFI1_CTRL_CTXT)
11294 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
77241056
MM		 11295 }
11296 if (op & HFI1_RCVCTRL_CTXT_DIS) {
11297 write_csr(dd, RCV_VL15, 0);
46b010d3
MB		 11298 /*
11299 * When receive context is being disabled turn on tail
11300 * update with a dummy tail address and then disable
11301 * receive context.
11302 */
11303 if (dd->rcvhdrtail_dummy_physaddr) {
11304 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11305 dd->rcvhdrtail_dummy_physaddr);
566c157c 11306 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
46b010d3
MB		 11307 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11308 }
11309
77241056
MM		 11310 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11311 }
11312 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
11313 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11314 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
11315 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11316 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
11317 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
566c157c
MH		 11318 if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11319 /* See comment on RcvCtxtCtrl.TailUpd above */
11320 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11321 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11322 }
77241056
MM		 11323 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11324 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11325 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11326 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11327 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
4d114fdd
JJ		 11328 /*
11329 * In one-packet-per-eager mode, the size comes from
11330 * the RcvArray entry.
11331 */
77241056
MM		 11332 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11333 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11334 }
11335 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11336 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11337 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11338 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11339 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11340 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11341 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11342 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11343 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
11344 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11345 rcd->rcvctrl = rcvctrl;
11346 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
11347 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
11348
11349 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
d0d236ea
JJ		 11350 if (did_enable &&
11351 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
77241056
MM		 11352 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11353 if (reg != 0) {
11354 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
17fb4f29 11355 ctxt, reg);
77241056
MM		 11356 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11357 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
11358 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
11359 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11360 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11361 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
17fb4f29 11362 ctxt, reg, reg == 0 ? "not" : "still");
77241056
MM		 11363 }
11364 }
11365
11366 if (did_enable) {
11367 /*
11368 * The interrupt timeout and count must be set after
11369 * the context is enabled to take effect.
11370 */
11371 /* set interrupt timeout */
11372 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
17fb4f29 11373 (u64)rcd->rcvavail_timeout <<
77241056
MM		 11374 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11375
11376 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
11377 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
11378 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11379 }
11380
11381 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
11382 /*
11383 * If the context has been disabled and the Tail Update has
46b010d3
MB		 11384 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
11385 * so it doesn't contain an address that is invalid.
77241056 11386 */
46b010d3
MB		 11387 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11388 dd->rcvhdrtail_dummy_physaddr);
77241056
MM		 11389}
11390
11391u32 hfi1_read_cntrs(struct hfi1_devdata *dd, loff_t pos, char **namep,
11392 u64 **cntrp)
11393{
11394 int ret;
11395 u64 val = 0;
11396
11397 if (namep) {
11398 ret = dd->cntrnameslen;
11399 if (pos != 0) {
11400 dd_dev_err(dd, "read_cntrs does not support indexing");
11401 return 0;
11402 }
11403 *namep = dd->cntrnames;
11404 } else {
11405 const struct cntr_entry *entry;
11406 int i, j;
11407
11408 ret = (dd->ndevcntrs) * sizeof(u64);
11409 if (pos != 0) {
11410 dd_dev_err(dd, "read_cntrs does not support indexing");
11411 return 0;
11412 }
11413
11414 /* Get the start of the block of counters */
11415 *cntrp = dd->cntrs;
11416
11417 /*
11418 * Now go and fill in each counter in the block.
11419 */
11420 for (i = 0; i < DEV_CNTR_LAST; i++) {
11421 entry = &dev_cntrs[i];
11422 hfi1_cdbg(CNTR, "reading %s", entry->name);
11423 if (entry->flags & CNTR_DISABLED) {
11424 /* Nothing */
11425 hfi1_cdbg(CNTR, "\tDisabled\n");
11426 } else {
11427 if (entry->flags & CNTR_VL) {
11428 hfi1_cdbg(CNTR, "\tPer VL\n");
11429 for (j = 0; j < C_VL_COUNT; j++) {
11430 val = entry->rw_cntr(entry,
11431 dd, j,
11432 CNTR_MODE_R,
11433 0);
11434 hfi1_cdbg(
11435 CNTR,
11436 "\t\tRead 0x%llx for %d\n",
11437 val, j);
11438 dd->cntrs[entry->offset + j] =
11439 val;
11440 }
a699c6c2
VM		 11441 } else if (entry->flags & CNTR_SDMA) {
11442 hfi1_cdbg(CNTR,
11443 "\t Per SDMA Engine\n");
11444 for (j = 0; j < dd->chip_sdma_engines;
11445 j++) {
11446 val =
11447 entry->rw_cntr(entry, dd, j,
11448 CNTR_MODE_R, 0);
11449 hfi1_cdbg(CNTR,
11450 "\t\tRead 0x%llx for %d\n",
11451 val, j);
11452 dd->cntrs[entry->offset + j] =
11453 val;
11454 }
77241056
MM		 11455 } else {
11456 val = entry->rw_cntr(entry, dd,
11457 CNTR_INVALID_VL,
11458 CNTR_MODE_R, 0);
11459 dd->cntrs[entry->offset] = val;
11460 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11461 }
11462 }
11463 }
11464 }
11465 return ret;
11466}
11467
11468/*
11469 * Used by sysfs to create files for hfi stats to read
11470 */
11471u32 hfi1_read_portcntrs(struct hfi1_devdata *dd, loff_t pos, u32 port,
11472 char **namep, u64 **cntrp)
11473{
11474 int ret;
11475 u64 val = 0;
11476
11477 if (namep) {
11478 ret = dd->portcntrnameslen;
11479 if (pos != 0) {
11480 dd_dev_err(dd, "index not supported");
11481 return 0;
11482 }
11483 *namep = dd->portcntrnames;
11484 } else {
11485 const struct cntr_entry *entry;
11486 struct hfi1_pportdata *ppd;
11487 int i, j;
11488
11489 ret = (dd->nportcntrs) * sizeof(u64);
11490 if (pos != 0) {
11491 dd_dev_err(dd, "indexing not supported");
11492 return 0;
11493 }
11494 ppd = (struct hfi1_pportdata *)(dd + 1 + port);
11495 *cntrp = ppd->cntrs;
11496
11497 for (i = 0; i < PORT_CNTR_LAST; i++) {
11498 entry = &port_cntrs[i];
11499 hfi1_cdbg(CNTR, "reading %s", entry->name);
11500 if (entry->flags & CNTR_DISABLED) {
11501 /* Nothing */
11502 hfi1_cdbg(CNTR, "\tDisabled\n");
11503 continue;
11504 }
11505
11506 if (entry->flags & CNTR_VL) {
11507 hfi1_cdbg(CNTR, "\tPer VL");
11508 for (j = 0; j < C_VL_COUNT; j++) {
11509 val = entry->rw_cntr(entry, ppd, j,
11510 CNTR_MODE_R,
11511 0);
11512 hfi1_cdbg(
11513 CNTR,
11514 "\t\tRead 0x%llx for %d",
11515 val, j);
11516 ppd->cntrs[entry->offset + j] = val;
11517 }
11518 } else {
11519 val = entry->rw_cntr(entry, ppd,
11520 CNTR_INVALID_VL,
11521 CNTR_MODE_R,
11522 0);
11523 ppd->cntrs[entry->offset] = val;
11524 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11525 }
11526 }
11527 }
11528 return ret;
11529}
11530
11531static void free_cntrs(struct hfi1_devdata *dd)
11532{
11533 struct hfi1_pportdata *ppd;
11534 int i;
11535
11536 if (dd->synth_stats_timer.data)
11537 del_timer_sync(&dd->synth_stats_timer);
11538 dd->synth_stats_timer.data = 0;
11539 ppd = (struct hfi1_pportdata *)(dd + 1);
11540 for (i = 0; i < dd->num_pports; i++, ppd++) {
11541 kfree(ppd->cntrs);
11542 kfree(ppd->scntrs);
4eb06882
DD		 11543 free_percpu(ppd->ibport_data.rvp.rc_acks);
11544 free_percpu(ppd->ibport_data.rvp.rc_qacks);
11545 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
77241056
MM		 11546 ppd->cntrs = NULL;
11547 ppd->scntrs = NULL;
4eb06882
DD		 11548 ppd->ibport_data.rvp.rc_acks = NULL;
11549 ppd->ibport_data.rvp.rc_qacks = NULL;
11550 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
77241056
MM		 11551 }
11552 kfree(dd->portcntrnames);
11553 dd->portcntrnames = NULL;
11554 kfree(dd->cntrs);
11555 dd->cntrs = NULL;
11556 kfree(dd->scntrs);
11557 dd->scntrs = NULL;
11558 kfree(dd->cntrnames);
11559 dd->cntrnames = NULL;
11560}
11561
11562#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
11563#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
11564
11565static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
11566 u64 *psval, void *context, int vl)
11567{
11568 u64 val;
11569 u64 sval = *psval;
11570
11571 if (entry->flags & CNTR_DISABLED) {
11572 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11573 return 0;
11574 }
11575
11576 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11577
11578 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
11579
11580 /* If its a synthetic counter there is more work we need to do */
11581 if (entry->flags & CNTR_SYNTH) {
11582 if (sval == CNTR_MAX) {
11583 /* No need to read already saturated */
11584 return CNTR_MAX;
11585 }
11586
11587 if (entry->flags & CNTR_32BIT) {
11588 /* 32bit counters can wrap multiple times */
11589 u64 upper = sval >> 32;
11590 u64 lower = (sval << 32) >> 32;
11591
11592 if (lower > val) { /* hw wrapped */
11593 if (upper == CNTR_32BIT_MAX)
11594 val = CNTR_MAX;
11595 else
11596 upper++;
11597 }
11598
11599 if (val != CNTR_MAX)
11600 val = (upper << 32) | val;
11601
11602 } else {
11603 /* If we rolled we are saturated */
11604 if ((val < sval) || (val > CNTR_MAX))
11605 val = CNTR_MAX;
11606 }
11607 }
11608
11609 *psval = val;
11610
11611 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11612
11613 return val;
11614}
11615
11616static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
11617 struct cntr_entry *entry,
11618 u64 *psval, void *context, int vl, u64 data)
11619{
11620 u64 val;
11621
11622 if (entry->flags & CNTR_DISABLED) {
11623 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11624 return 0;
11625 }
11626
11627 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11628
11629 if (entry->flags & CNTR_SYNTH) {
11630 *psval = data;
11631 if (entry->flags & CNTR_32BIT) {
11632 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11633 (data << 32) >> 32);
11634 val = data; /* return the full 64bit value */
11635 } else {
11636 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11637 data);
11638 }
11639 } else {
11640 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
11641 }
11642
11643 *psval = val;
11644
11645 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11646
11647 return val;
11648}
11649
11650u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
11651{
11652 struct cntr_entry *entry;
11653 u64 *sval;
11654
11655 entry = &dev_cntrs[index];
11656 sval = dd->scntrs + entry->offset;
11657
11658 if (vl != CNTR_INVALID_VL)
11659 sval += vl;
11660
11661 return read_dev_port_cntr(dd, entry, sval, dd, vl);
11662}
11663
11664u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
11665{
11666 struct cntr_entry *entry;
11667 u64 *sval;
11668
11669 entry = &dev_cntrs[index];
11670 sval = dd->scntrs + entry->offset;
11671
11672 if (vl != CNTR_INVALID_VL)
11673 sval += vl;
11674
11675 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
11676}
11677
11678u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
11679{
11680 struct cntr_entry *entry;
11681 u64 *sval;
11682
11683 entry = &port_cntrs[index];
11684 sval = ppd->scntrs + entry->offset;
11685
11686 if (vl != CNTR_INVALID_VL)
11687 sval += vl;
11688
11689 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11690 (index <= C_RCV_HDR_OVF_LAST)) {
11691 /* We do not want to bother for disabled contexts */
11692 return 0;
11693 }
11694
11695 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
11696}
11697
11698u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
11699{
11700 struct cntr_entry *entry;
11701 u64 *sval;
11702
11703 entry = &port_cntrs[index];
11704 sval = ppd->scntrs + entry->offset;
11705
11706 if (vl != CNTR_INVALID_VL)
11707 sval += vl;
11708
11709 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11710 (index <= C_RCV_HDR_OVF_LAST)) {
11711 /* We do not want to bother for disabled contexts */
11712 return 0;
11713 }
11714
11715 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
11716}
11717
11718static void update_synth_timer(unsigned long opaque)
11719{
11720 u64 cur_tx;
11721 u64 cur_rx;
11722 u64 total_flits;
11723 u8 update = 0;
11724 int i, j, vl;
11725 struct hfi1_pportdata *ppd;
11726 struct cntr_entry *entry;
11727
11728 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
11729
11730 /*
11731 * Rather than keep beating on the CSRs pick a minimal set that we can
11732 * check to watch for potential roll over. We can do this by looking at
11733 * the number of flits sent/recv. If the total flits exceeds 32bits then
11734 * we have to iterate all the counters and update.
11735 */
11736 entry = &dev_cntrs[C_DC_RCV_FLITS];
11737 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11738
11739 entry = &dev_cntrs[C_DC_XMIT_FLITS];
11740 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11741
11742 hfi1_cdbg(
11743 CNTR,
11744 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
11745 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
11746
11747 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
11748 /*
11749 * May not be strictly necessary to update but it won't hurt and
11750 * simplifies the logic here.
11751 */
11752 update = 1;
11753 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
11754 dd->unit);
11755 } else {
11756 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
11757 hfi1_cdbg(CNTR,
11758 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
11759 total_flits, (u64)CNTR_32BIT_MAX);
11760 if (total_flits >= CNTR_32BIT_MAX) {
11761 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
11762 dd->unit);
11763 update = 1;
11764 }
11765 }
11766
11767 if (update) {
11768 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
11769 for (i = 0; i < DEV_CNTR_LAST; i++) {
11770 entry = &dev_cntrs[i];
11771 if (entry->flags & CNTR_VL) {
11772 for (vl = 0; vl < C_VL_COUNT; vl++)
11773 read_dev_cntr(dd, i, vl);
11774 } else {
11775 read_dev_cntr(dd, i, CNTR_INVALID_VL);
11776 }
11777 }
11778 ppd = (struct hfi1_pportdata *)(dd + 1);
11779 for (i = 0; i < dd->num_pports; i++, ppd++) {
11780 for (j = 0; j < PORT_CNTR_LAST; j++) {
11781 entry = &port_cntrs[j];
11782 if (entry->flags & CNTR_VL) {
11783 for (vl = 0; vl < C_VL_COUNT; vl++)
11784 read_port_cntr(ppd, j, vl);
11785 } else {
11786 read_port_cntr(ppd, j, CNTR_INVALID_VL);
11787 }
11788 }
11789 }
11790
11791 /*
11792 * We want the value in the register. The goal is to keep track
11793 * of the number of "ticks" not the counter value. In other
11794 * words if the register rolls we want to notice it and go ahead
11795 * and force an update.
11796 */
11797 entry = &dev_cntrs[C_DC_XMIT_FLITS];
11798 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11799 CNTR_MODE_R, 0);
11800
11801 entry = &dev_cntrs[C_DC_RCV_FLITS];
11802 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11803 CNTR_MODE_R, 0);
11804
11805 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
11806 dd->unit, dd->last_tx, dd->last_rx);
11807
11808 } else {
11809 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
11810 }
11811
11812mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
11813}
11814
11815#define C_MAX_NAME 13 /* 12 chars + one for /0 */
11816static int init_cntrs(struct hfi1_devdata *dd)
11817{
c024c554 11818 int i, rcv_ctxts, j;
77241056
MM		 11819 size_t sz;
11820 char *p;
11821 char name[C_MAX_NAME];
11822 struct hfi1_pportdata *ppd;
11d2b114
SS		 11823 const char *bit_type_32 = ",32";
11824 const int bit_type_32_sz = strlen(bit_type_32);
77241056
MM		 11825
11826 /* set up the stats timer; the add_timer is done at the end */
24523a94
MFW		 11827 setup_timer(&dd->synth_stats_timer, update_synth_timer,
11828 (unsigned long)dd);
77241056
MM		 11829
11830 /***********************/
11831 /* per device counters */
11832 /***********************/
11833
11834 /* size names and determine how many we have*/
11835 dd->ndevcntrs = 0;
11836 sz = 0;
77241056
MM		 11837
11838 for (i = 0; i < DEV_CNTR_LAST; i++) {
77241056
MM		 11839 if (dev_cntrs[i].flags & CNTR_DISABLED) {
11840 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
11841 continue;
11842 }
11843
11844 if (dev_cntrs[i].flags & CNTR_VL) {
c024c554 11845 dev_cntrs[i].offset = dd->ndevcntrs;
77241056 11846 for (j = 0; j < C_VL_COUNT; j++) {
77241056 11847 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 11848 dev_cntrs[i].name, vl_from_idx(j));
77241056 11849 sz += strlen(name);
11d2b114
SS		 11850 /* Add ",32" for 32-bit counters */
11851 if (dev_cntrs[i].flags & CNTR_32BIT)
11852 sz += bit_type_32_sz;
77241056 11853 sz++;
77241056 11854 dd->ndevcntrs++;
77241056 11855 }
a699c6c2 11856 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
c024c554 11857 dev_cntrs[i].offset = dd->ndevcntrs;
a699c6c2 11858 for (j = 0; j < dd->chip_sdma_engines; j++) {
a699c6c2
VM		 11859 snprintf(name, C_MAX_NAME, "%s%d",
11860 dev_cntrs[i].name, j);
11861 sz += strlen(name);
11d2b114
SS		 11862 /* Add ",32" for 32-bit counters */
11863 if (dev_cntrs[i].flags & CNTR_32BIT)
11864 sz += bit_type_32_sz;
a699c6c2 11865 sz++;
a699c6c2 11866 dd->ndevcntrs++;
a699c6c2 11867 }
77241056 11868 } else {
11d2b114 11869 /* +1 for newline. */
77241056 11870 sz += strlen(dev_cntrs[i].name) + 1;
11d2b114
SS		 11871 /* Add ",32" for 32-bit counters */
11872 if (dev_cntrs[i].flags & CNTR_32BIT)
11873 sz += bit_type_32_sz;
c024c554 11874 dev_cntrs[i].offset = dd->ndevcntrs;
77241056 11875 dd->ndevcntrs++;
77241056
MM		 11876 }
11877 }
11878
11879 /* allocate space for the counter values */
c024c554 11880 dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
77241056
MM		 11881 if (!dd->cntrs)
11882 goto bail;
11883
c024c554 11884 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
77241056
MM		 11885 if (!dd->scntrs)
11886 goto bail;
11887
77241056
MM		 11888 /* allocate space for the counter names */
11889 dd->cntrnameslen = sz;
11890 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
11891 if (!dd->cntrnames)
11892 goto bail;
11893
11894 /* fill in the names */
c024c554 11895 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
77241056
MM		 11896 if (dev_cntrs[i].flags & CNTR_DISABLED) {
11897 /* Nothing */
11d2b114
SS		 11898 } else if (dev_cntrs[i].flags & CNTR_VL) {
11899 for (j = 0; j < C_VL_COUNT; j++) {
11d2b114
SS		 11900 snprintf(name, C_MAX_NAME, "%s%d",
11901 dev_cntrs[i].name,
11902 vl_from_idx(j));
11903 memcpy(p, name, strlen(name));
11904 p += strlen(name);
11905
11906 /* Counter is 32 bits */
11907 if (dev_cntrs[i].flags & CNTR_32BIT) {
11908 memcpy(p, bit_type_32, bit_type_32_sz);
11909 p += bit_type_32_sz;
77241056 11910 }
11d2b114
SS		 11911
11912 *p++ = '\n';
11913 }
11914 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
11915 for (j = 0; j < dd->chip_sdma_engines; j++) {
11d2b114
SS		 11916 snprintf(name, C_MAX_NAME, "%s%d",
11917 dev_cntrs[i].name, j);
11918 memcpy(p, name, strlen(name));
11919 p += strlen(name);
11920
11921 /* Counter is 32 bits */
11922 if (dev_cntrs[i].flags & CNTR_32BIT) {
11923 memcpy(p, bit_type_32, bit_type_32_sz);
11924 p += bit_type_32_sz;
a699c6c2 11925 }
11d2b114 11926
77241056
MM		 11927 *p++ = '\n';
11928 }
11d2b114
SS		 11929 } else {
11930 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
11931 p += strlen(dev_cntrs[i].name);
11932
11933 /* Counter is 32 bits */
11934 if (dev_cntrs[i].flags & CNTR_32BIT) {
11935 memcpy(p, bit_type_32, bit_type_32_sz);
11936 p += bit_type_32_sz;
11937 }
11938
11939 *p++ = '\n';
77241056
MM		 11940 }
11941 }
11942
11943 /*********************/
11944 /* per port counters */
11945 /*********************/
11946
11947 /*
11948 * Go through the counters for the overflows and disable the ones we
11949 * don't need. This varies based on platform so we need to do it
11950 * dynamically here.
11951 */
11952 rcv_ctxts = dd->num_rcv_contexts;
11953 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
11954 i <= C_RCV_HDR_OVF_LAST; i++) {
11955 port_cntrs[i].flags |= CNTR_DISABLED;
11956 }
11957
11958 /* size port counter names and determine how many we have*/
11959 sz = 0;
11960 dd->nportcntrs = 0;
11961 for (i = 0; i < PORT_CNTR_LAST; i++) {
77241056
MM		 11962 if (port_cntrs[i].flags & CNTR_DISABLED) {
11963 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
11964 continue;
11965 }
11966
11967 if (port_cntrs[i].flags & CNTR_VL) {
77241056
MM		 11968 port_cntrs[i].offset = dd->nportcntrs;
11969 for (j = 0; j < C_VL_COUNT; j++) {
77241056 11970 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 11971 port_cntrs[i].name, vl_from_idx(j));
77241056 11972 sz += strlen(name);
11d2b114
SS		 11973 /* Add ",32" for 32-bit counters */
11974 if (port_cntrs[i].flags & CNTR_32BIT)
11975 sz += bit_type_32_sz;
77241056 11976 sz++;
77241056
MM		 11977 dd->nportcntrs++;
11978 }
11979 } else {
11d2b114 11980 /* +1 for newline */
77241056 11981 sz += strlen(port_cntrs[i].name) + 1;
11d2b114
SS		 11982 /* Add ",32" for 32-bit counters */
11983 if (port_cntrs[i].flags & CNTR_32BIT)
11984 sz += bit_type_32_sz;
77241056
MM		 11985 port_cntrs[i].offset = dd->nportcntrs;
11986 dd->nportcntrs++;
77241056
MM		 11987 }
11988 }
11989
11990 /* allocate space for the counter names */
11991 dd->portcntrnameslen = sz;
11992 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
11993 if (!dd->portcntrnames)
11994 goto bail;
11995
11996 /* fill in port cntr names */
11997 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
11998 if (port_cntrs[i].flags & CNTR_DISABLED)
11999 continue;
12000
12001 if (port_cntrs[i].flags & CNTR_VL) {
12002 for (j = 0; j < C_VL_COUNT; j++) {
77241056 12003 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 12004 port_cntrs[i].name, vl_from_idx(j));
77241056
MM		 12005 memcpy(p, name, strlen(name));
12006 p += strlen(name);
11d2b114
SS		 12007
12008 /* Counter is 32 bits */
12009 if (port_cntrs[i].flags & CNTR_32BIT) {
12010 memcpy(p, bit_type_32, bit_type_32_sz);
12011 p += bit_type_32_sz;
12012 }
12013
77241056
MM		 12014 *p++ = '\n';
12015 }
12016 } else {
12017 memcpy(p, port_cntrs[i].name,
12018 strlen(port_cntrs[i].name));
12019 p += strlen(port_cntrs[i].name);
11d2b114
SS		 12020
12021 /* Counter is 32 bits */
12022 if (port_cntrs[i].flags & CNTR_32BIT) {
12023 memcpy(p, bit_type_32, bit_type_32_sz);
12024 p += bit_type_32_sz;
12025 }
12026
77241056
MM		 12027 *p++ = '\n';
12028 }
12029 }
12030
12031 /* allocate per port storage for counter values */
12032 ppd = (struct hfi1_pportdata *)(dd + 1);
12033 for (i = 0; i < dd->num_pports; i++, ppd++) {
12034 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12035 if (!ppd->cntrs)
12036 goto bail;
12037
12038 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12039 if (!ppd->scntrs)
12040 goto bail;
12041 }
12042
12043 /* CPU counters need to be allocated and zeroed */
12044 if (init_cpu_counters(dd))
12045 goto bail;
12046
12047 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12048 return 0;
12049bail:
12050 free_cntrs(dd);
12051 return -ENOMEM;
12052}
12053
77241056
MM		 12054static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12055{
12056 switch (chip_lstate) {
12057 default:
12058 dd_dev_err(dd,
17fb4f29
JJ		 12059 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12060 chip_lstate);
77241056
MM		 12061 /* fall through */
12062 case LSTATE_DOWN:
12063 return IB_PORT_DOWN;
12064 case LSTATE_INIT:
12065 return IB_PORT_INIT;
12066 case LSTATE_ARMED:
12067 return IB_PORT_ARMED;
12068 case LSTATE_ACTIVE:
12069 return IB_PORT_ACTIVE;
12070 }
12071}
12072
12073u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12074{
12075 /* look at the HFI meta-states only */
12076 switch (chip_pstate & 0xf0) {
12077 default:
12078 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
17fb4f29 12079 chip_pstate);
77241056
MM		 12080 /* fall through */
12081 case PLS_DISABLED:
12082 return IB_PORTPHYSSTATE_DISABLED;
12083 case PLS_OFFLINE:
12084 return OPA_PORTPHYSSTATE_OFFLINE;
12085 case PLS_POLLING:
12086 return IB_PORTPHYSSTATE_POLLING;
12087 case PLS_CONFIGPHY:
12088 return IB_PORTPHYSSTATE_TRAINING;
12089 case PLS_LINKUP:
12090 return IB_PORTPHYSSTATE_LINKUP;
12091 case PLS_PHYTEST:
12092 return IB_PORTPHYSSTATE_PHY_TEST;
12093 }
12094}
12095
12096/* return the OPA port logical state name */
12097const char *opa_lstate_name(u32 lstate)
12098{
12099 static const char * const port_logical_names[] = {
12100 "PORT_NOP",
12101 "PORT_DOWN",
12102 "PORT_INIT",
12103 "PORT_ARMED",
12104 "PORT_ACTIVE",
12105 "PORT_ACTIVE_DEFER",
12106 };
12107 if (lstate < ARRAY_SIZE(port_logical_names))
12108 return port_logical_names[lstate];
12109 return "unknown";
12110}
12111
12112/* return the OPA port physical state name */
12113const char *opa_pstate_name(u32 pstate)
12114{
12115 static const char * const port_physical_names[] = {
12116 "PHYS_NOP",
12117 "reserved1",
12118 "PHYS_POLL",
12119 "PHYS_DISABLED",
12120 "PHYS_TRAINING",
12121 "PHYS_LINKUP",
12122 "PHYS_LINK_ERR_RECOVER",
12123 "PHYS_PHY_TEST",
12124 "reserved8",
12125 "PHYS_OFFLINE",
12126 "PHYS_GANGED",
12127 "PHYS_TEST",
12128 };
12129 if (pstate < ARRAY_SIZE(port_physical_names))
12130 return port_physical_names[pstate];
12131 return "unknown";
12132}
12133
12134/*
12135 * Read the hardware link state and set the driver's cached value of it.
12136 * Return the (new) current value.
12137 */
12138u32 get_logical_state(struct hfi1_pportdata *ppd)
12139{
12140 u32 new_state;
12141
12142 new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
12143 if (new_state != ppd->lstate) {
12144 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
17fb4f29 12145 opa_lstate_name(new_state), new_state);
77241056
MM		 12146 ppd->lstate = new_state;
12147 }
12148 /*
12149 * Set port status flags in the page mapped into userspace
12150 * memory. Do it here to ensure a reliable state - this is
12151 * the only function called by all state handling code.
12152 * Always set the flags due to the fact that the cache value
12153 * might have been changed explicitly outside of this
12154 * function.
12155 */
12156 if (ppd->statusp) {
12157 switch (ppd->lstate) {
12158 case IB_PORT_DOWN:
12159 case IB_PORT_INIT:
12160 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12161 HFI1_STATUS_IB_READY);
12162 break;
12163 case IB_PORT_ARMED:
12164 *ppd->statusp |= HFI1_STATUS_IB_CONF;
12165 break;
12166 case IB_PORT_ACTIVE:
12167 *ppd->statusp |= HFI1_STATUS_IB_READY;
12168 break;
12169 }
12170 }
12171 return ppd->lstate;
12172}
12173
12174/**
12175 * wait_logical_linkstate - wait for an IB link state change to occur
12176 * @ppd: port device
12177 * @state: the state to wait for
12178 * @msecs: the number of milliseconds to wait
12179 *
12180 * Wait up to msecs milliseconds for IB link state change to occur.
12181 * For now, take the easy polling route.
12182 * Returns 0 if state reached, otherwise -ETIMEDOUT.
12183 */
12184static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12185 int msecs)
12186{
12187 unsigned long timeout;
12188
12189 timeout = jiffies + msecs_to_jiffies(msecs);
12190 while (1) {
12191 if (get_logical_state(ppd) == state)
12192 return 0;
12193 if (time_after(jiffies, timeout))
12194 break;
12195 msleep(20);
12196 }
12197 dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
12198
12199 return -ETIMEDOUT;
12200}
12201
12202u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
12203{
77241056
MM		 12204 u32 pstate;
12205 u32 ib_pstate;
12206
12207 pstate = read_physical_state(ppd->dd);
12208 ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
f45c8dc8 12209 if (ppd->last_pstate != ib_pstate) {
77241056 12210 dd_dev_info(ppd->dd,
17fb4f29
JJ		 12211 "%s: physical state changed to %s (0x%x), phy 0x%x\n",
12212 __func__, opa_pstate_name(ib_pstate), ib_pstate,
12213 pstate);
f45c8dc8 12214 ppd->last_pstate = ib_pstate;
77241056
MM		 12215 }
12216 return ib_pstate;
12217}
12218
12219/*
12220 * Read/modify/write ASIC_QSFP register bits as selected by mask
12221 * data: 0 or 1 in the positions depending on what needs to be written
12222 * dir: 0 for read, 1 for write
12223 * mask: select by setting
12224 * I2CCLK (bit 0)
12225 * I2CDATA (bit 1)
12226 */
12227u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
12228 u32 mask)
12229{
12230 u64 qsfp_oe, target_oe;
12231
12232 target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
12233 if (mask) {
12234 /* We are writing register bits, so lock access */
12235 dir &= mask;
12236 data &= mask;
12237
12238 qsfp_oe = read_csr(dd, target_oe);
12239 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
12240 write_csr(dd, target_oe, qsfp_oe);
12241 }
12242 /* We are exclusively reading bits here, but it is unlikely
12243 * we'll get valid data when we set the direction of the pin
12244 * in the same call, so read should call this function again
12245 * to get valid data
12246 */
12247 return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
12248}
12249
12250#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12251(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12252
12253#define SET_STATIC_RATE_CONTROL_SMASK(r) \
12254(r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12255
12256int hfi1_init_ctxt(struct send_context *sc)
12257{
d125a6c6 12258 if (sc) {
77241056
MM		 12259 struct hfi1_devdata *dd = sc->dd;
12260 u64 reg;
12261 u8 set = (sc->type == SC_USER ?
12262 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
12263 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
12264 reg = read_kctxt_csr(dd, sc->hw_context,
12265 SEND_CTXT_CHECK_ENABLE);
12266 if (set)
12267 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
12268 else
12269 SET_STATIC_RATE_CONTROL_SMASK(reg);
12270 write_kctxt_csr(dd, sc->hw_context,
12271 SEND_CTXT_CHECK_ENABLE, reg);
12272 }
12273 return 0;
12274}
12275
12276int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
12277{
12278 int ret = 0;
12279 u64 reg;
12280
12281 if (dd->icode != ICODE_RTL_SILICON) {
12282 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
12283 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
12284 __func__);
12285 return -EINVAL;
12286 }
12287 reg = read_csr(dd, ASIC_STS_THERM);
12288 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
12289 ASIC_STS_THERM_CURR_TEMP_MASK);
12290 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
12291 ASIC_STS_THERM_LO_TEMP_MASK);
12292 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
12293 ASIC_STS_THERM_HI_TEMP_MASK);
12294 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
12295 ASIC_STS_THERM_CRIT_TEMP_MASK);
12296 /* triggers is a 3-bit value - 1 bit per trigger. */
12297 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
12298
12299 return ret;
12300}
12301
12302/* ========================================================================= */
12303
12304/*
12305 * Enable/disable chip from delivering interrupts.
12306 */
12307void set_intr_state(struct hfi1_devdata *dd, u32 enable)
12308{
12309 int i;
12310
12311 /*
12312 * In HFI, the mask needs to be 1 to allow interrupts.
12313 */
12314 if (enable) {
77241056
MM		 12315 /* enable all interrupts */
12316 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12317 write_csr(dd, CCE_INT_MASK + (8 * i), ~(u64)0);
77241056 12318
8ebd4cf1 12319 init_qsfp_int(dd);
77241056
MM		 12320 } else {
12321 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12322 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
77241056
MM		 12323 }
12324}
12325
12326/*
12327 * Clear all interrupt sources on the chip.
12328 */
12329static void clear_all_interrupts(struct hfi1_devdata *dd)
12330{
12331 int i;
12332
12333 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12334 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
77241056
MM		 12335
12336 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
12337 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
12338 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
12339 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
12340 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
12341 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
12342 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
12343 for (i = 0; i < dd->chip_send_contexts; i++)
12344 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
12345 for (i = 0; i < dd->chip_sdma_engines; i++)
12346 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
12347
12348 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
12349 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
12350 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
12351}
12352
12353/* Move to pcie.c? */
12354static void disable_intx(struct pci_dev *pdev)
12355{
12356 pci_intx(pdev, 0);
12357}
12358
12359static void clean_up_interrupts(struct hfi1_devdata *dd)
12360{
12361 int i;
12362
12363 /* remove irqs - must happen before disabling/turning off */
12364 if (dd->num_msix_entries) {
12365 /* MSI-X */
12366 struct hfi1_msix_entry *me = dd->msix_entries;
12367
12368 for (i = 0; i < dd->num_msix_entries; i++, me++) {
d125a6c6 12369 if (!me->arg) /* => no irq, no affinity */
957558c9
MH		 12370 continue;
12371 hfi1_put_irq_affinity(dd, &dd->msix_entries[i]);
77241056
MM		 12372 free_irq(me->msix.vector, me->arg);
12373 }
12374 } else {
12375 /* INTx */
12376 if (dd->requested_intx_irq) {
12377 free_irq(dd->pcidev->irq, dd);
12378 dd->requested_intx_irq = 0;
12379 }
12380 }
12381
12382 /* turn off interrupts */
12383 if (dd->num_msix_entries) {
12384 /* MSI-X */
6e5b6131 12385 pci_disable_msix(dd->pcidev);
77241056
MM		 12386 } else {
12387 /* INTx */
12388 disable_intx(dd->pcidev);
12389 }
12390
12391 /* clean structures */
77241056
MM		 12392 kfree(dd->msix_entries);
12393 dd->msix_entries = NULL;
12394 dd->num_msix_entries = 0;
12395}
12396
12397/*
12398 * Remap the interrupt source from the general handler to the given MSI-X
12399 * interrupt.
12400 */
12401static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
12402{
12403 u64 reg;
12404 int m, n;
12405
12406 /* clear from the handled mask of the general interrupt */
12407 m = isrc / 64;
12408 n = isrc % 64;
12409 dd->gi_mask[m] &= ~((u64)1 << n);
12410
12411 /* direct the chip source to the given MSI-X interrupt */
12412 m = isrc / 8;
12413 n = isrc % 8;
8638b77f
JJ		 12414 reg = read_csr(dd, CCE_INT_MAP + (8 * m));
12415 reg &= ~((u64)0xff << (8 * n));
12416 reg |= ((u64)msix_intr & 0xff) << (8 * n);
12417 write_csr(dd, CCE_INT_MAP + (8 * m), reg);
77241056
MM		 12418}
12419
12420static void remap_sdma_interrupts(struct hfi1_devdata *dd,
12421 int engine, int msix_intr)
12422{
12423 /*
12424 * SDMA engine interrupt sources grouped by type, rather than
12425 * engine. Per-engine interrupts are as follows:
12426 * SDMA
12427 * SDMAProgress
12428 * SDMAIdle
12429 */
8638b77f 12430 remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12431 msix_intr);
8638b77f 12432 remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12433 msix_intr);
8638b77f 12434 remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12435 msix_intr);
77241056
MM		 12436}
12437
77241056
MM		 12438static int request_intx_irq(struct hfi1_devdata *dd)
12439{
12440 int ret;
12441
9805071e
JJ		 12442 snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
12443 dd->unit);
77241056 12444 ret = request_irq(dd->pcidev->irq, general_interrupt,
17fb4f29 12445 IRQF_SHARED, dd->intx_name, dd);
77241056
MM		 12446 if (ret)
12447 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
17fb4f29 12448 ret);
77241056
MM		 12449 else
12450 dd->requested_intx_irq = 1;
12451 return ret;
12452}
12453
12454static int request_msix_irqs(struct hfi1_devdata *dd)
12455{
77241056
MM		 12456 int first_general, last_general;
12457 int first_sdma, last_sdma;
12458 int first_rx, last_rx;
957558c9 12459 int i, ret = 0;
77241056
MM		 12460
12461 /* calculate the ranges we are going to use */
12462 first_general = 0;
f3ff8189
JJ		 12463 last_general = first_general + 1;
12464 first_sdma = last_general;
12465 last_sdma = first_sdma + dd->num_sdma;
12466 first_rx = last_sdma;
77241056
MM		 12467 last_rx = first_rx + dd->n_krcv_queues;
12468
77241056
MM		 12469 /*
12470 * Sanity check - the code expects all SDMA chip source
12471 * interrupts to be in the same CSR, starting at bit 0. Verify
12472 * that this is true by checking the bit location of the start.
12473 */
12474 BUILD_BUG_ON(IS_SDMA_START % 64);
12475
12476 for (i = 0; i < dd->num_msix_entries; i++) {
12477 struct hfi1_msix_entry *me = &dd->msix_entries[i];
12478 const char *err_info;
12479 irq_handler_t handler;
f4f30031 12480 irq_handler_t thread = NULL;
77241056
MM		 12481 void *arg;
12482 int idx;
12483 struct hfi1_ctxtdata *rcd = NULL;
12484 struct sdma_engine *sde = NULL;
12485
12486 /* obtain the arguments to request_irq */
12487 if (first_general <= i && i < last_general) {
12488 idx = i - first_general;
12489 handler = general_interrupt;
12490 arg = dd;
12491 snprintf(me->name, sizeof(me->name),
9805071e 12492 DRIVER_NAME "_%d", dd->unit);
77241056 12493 err_info = "general";
957558c9 12494 me->type = IRQ_GENERAL;
77241056
MM		 12495 } else if (first_sdma <= i && i < last_sdma) {
12496 idx = i - first_sdma;
12497 sde = &dd->per_sdma[idx];
12498 handler = sdma_interrupt;
12499 arg = sde;
12500 snprintf(me->name, sizeof(me->name),
9805071e 12501 DRIVER_NAME "_%d sdma%d", dd->unit, idx);
77241056
MM		 12502 err_info = "sdma";
12503 remap_sdma_interrupts(dd, idx, i);
957558c9 12504 me->type = IRQ_SDMA;
77241056
MM		 12505 } else if (first_rx <= i && i < last_rx) {
12506 idx = i - first_rx;
12507 rcd = dd->rcd[idx];
12508 /* no interrupt if no rcd */
12509 if (!rcd)
12510 continue;
12511 /*
12512 * Set the interrupt register and mask for this
12513 * context's interrupt.
12514 */
8638b77f 12515 rcd->ireg = (IS_RCVAVAIL_START + idx) / 64;
77241056 12516 rcd->imask = ((u64)1) <<
8638b77f 12517 ((IS_RCVAVAIL_START + idx) % 64);
77241056 12518 handler = receive_context_interrupt;
f4f30031 12519 thread = receive_context_thread;
77241056
MM		 12520 arg = rcd;
12521 snprintf(me->name, sizeof(me->name),
9805071e 12522 DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
77241056 12523 err_info = "receive context";
66c0933b 12524 remap_intr(dd, IS_RCVAVAIL_START + idx, i);
957558c9 12525 me->type = IRQ_RCVCTXT;
77241056
MM		 12526 } else {
12527 /* not in our expected range - complain, then
4d114fdd
JJ		 12528 * ignore it
12529 */
77241056 12530 dd_dev_err(dd,
17fb4f29 12531 "Unexpected extra MSI-X interrupt %d\n", i);
77241056
MM		 12532 continue;
12533 }
12534 /* no argument, no interrupt */
d125a6c6 12535 if (!arg)
77241056
MM		 12536 continue;
12537 /* make sure the name is terminated */
8638b77f 12538 me->name[sizeof(me->name) - 1] = 0;
77241056 12539
f4f30031 12540 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
17fb4f29 12541 me->name, arg);
77241056
MM		 12542 if (ret) {
12543 dd_dev_err(dd,
17fb4f29
JJ		 12544 "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
12545 err_info, me->msix.vector, idx, ret);
77241056
MM		 12546 return ret;
12547 }
12548 /*
12549 * assign arg after request_irq call, so it will be
12550 * cleaned up
12551 */
12552 me->arg = arg;
12553
957558c9
MH		 12554 ret = hfi1_get_irq_affinity(dd, me);
12555 if (ret)
12556 dd_dev_err(dd,
12557 "unable to pin IRQ %d\n", ret);
77241056
MM		 12558 }
12559
77241056 12560 return ret;
77241056
MM		 12561}
12562
12563/*
12564 * Set the general handler to accept all interrupts, remap all
12565 * chip interrupts back to MSI-X 0.
12566 */
12567static void reset_interrupts(struct hfi1_devdata *dd)
12568{
12569 int i;
12570
12571 /* all interrupts handled by the general handler */
12572 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12573 dd->gi_mask[i] = ~(u64)0;
12574
12575 /* all chip interrupts map to MSI-X 0 */
12576 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
8638b77f 12577 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
77241056
MM		 12578}
12579
12580static int set_up_interrupts(struct hfi1_devdata *dd)
12581{
12582 struct hfi1_msix_entry *entries;
12583 u32 total, request;
12584 int i, ret;
12585 int single_interrupt = 0; /* we expect to have all the interrupts */
12586
12587 /*
12588 * Interrupt count:
12589 * 1 general, "slow path" interrupt (includes the SDMA engines
12590 * slow source, SDMACleanupDone)
12591 * N interrupts - one per used SDMA engine
12592 * M interrupt - one per kernel receive context
12593 */
12594 total = 1 + dd->num_sdma + dd->n_krcv_queues;
12595
12596 entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
12597 if (!entries) {
77241056
MM		 12598 ret = -ENOMEM;
12599 goto fail;
12600 }
12601 /* 1-1 MSI-X entry assignment */
12602 for (i = 0; i < total; i++)
12603 entries[i].msix.entry = i;
12604
12605 /* ask for MSI-X interrupts */
12606 request = total;
12607 request_msix(dd, &request, entries);
12608
12609 if (request == 0) {
12610 /* using INTx */
12611 /* dd->num_msix_entries already zero */
12612 kfree(entries);
12613 single_interrupt = 1;
12614 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
12615 } else {
12616 /* using MSI-X */
12617 dd->num_msix_entries = request;
12618 dd->msix_entries = entries;
12619
12620 if (request != total) {
12621 /* using MSI-X, with reduced interrupts */
12622 dd_dev_err(
12623 dd,
12624 "cannot handle reduced interrupt case, want %u, got %u\n",
12625 total, request);
12626 ret = -EINVAL;
12627 goto fail;
12628 }
12629 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
12630 }
12631
12632 /* mask all interrupts */
12633 set_intr_state(dd, 0);
12634 /* clear all pending interrupts */
12635 clear_all_interrupts(dd);
12636
12637 /* reset general handler mask, chip MSI-X mappings */
12638 reset_interrupts(dd);
12639
12640 if (single_interrupt)
12641 ret = request_intx_irq(dd);
12642 else
12643 ret = request_msix_irqs(dd);
12644 if (ret)
12645 goto fail;
12646
12647 return 0;
12648
12649fail:
12650 clean_up_interrupts(dd);
12651 return ret;
12652}
12653
12654/*
12655 * Set up context values in dd. Sets:
12656 *
12657 * num_rcv_contexts - number of contexts being used
12658 * n_krcv_queues - number of kernel contexts
12659 * first_user_ctxt - first non-kernel context in array of contexts
12660 * freectxts - number of free user contexts
12661 * num_send_contexts - number of PIO send contexts being used
12662 */
12663static int set_up_context_variables(struct hfi1_devdata *dd)
12664{
12665 int num_kernel_contexts;
77241056
MM		 12666 int total_contexts;
12667 int ret;
12668 unsigned ngroups;
12669
12670 /*
12671 * Kernel contexts: (to be fixed later):
12672 * - min or 2 or 1 context/numa
82c2611d
NV		 12673 * - Context 0 - control context (VL15/multicast/error)
12674 * - Context 1 - default context
77241056
MM		 12675 */
12676 if (n_krcvqs)
82c2611d
NV		 12677 /*
12678 * Don't count context 0 in n_krcvqs since
12679 * is isn't used for normal verbs traffic.
12680 *
12681 * krcvqs will reflect number of kernel
12682 * receive contexts above 0.
12683 */
12684 num_kernel_contexts = n_krcvqs + MIN_KERNEL_KCTXTS - 1;
77241056 12685 else
0edf80ea 12686 num_kernel_contexts = num_online_nodes() + 1;
77241056
MM		 12687 num_kernel_contexts =
12688 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
12689 /*
12690 * Every kernel receive context needs an ACK send context.
12691 * one send context is allocated for each VL{0-7} and VL15
12692 */
12693 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
12694 dd_dev_err(dd,
12695 "Reducing # kernel rcv contexts to: %d, from %d\n",
12696 (int)(dd->chip_send_contexts - num_vls - 1),
12697 (int)num_kernel_contexts);
12698 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
12699 }
12700 /*
12701 * User contexts: (to be fixed later)
2ce6bf22
SS		 12702 * - default to 1 user context per CPU if num_user_contexts is
12703 * negative
77241056 12704 */
2ce6bf22 12705 if (num_user_contexts < 0)
77241056
MM		 12706 num_user_contexts = num_online_cpus();
12707
12708 total_contexts = num_kernel_contexts + num_user_contexts;
12709
12710 /*
12711 * Adjust the counts given a global max.
12712 */
12713 if (total_contexts > dd->chip_rcv_contexts) {
12714 dd_dev_err(dd,
12715 "Reducing # user receive contexts to: %d, from %d\n",
12716 (int)(dd->chip_rcv_contexts - num_kernel_contexts),
12717 (int)num_user_contexts);
12718 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
12719 /* recalculate */
12720 total_contexts = num_kernel_contexts + num_user_contexts;
12721 }
12722
12723 /* the first N are kernel contexts, the rest are user contexts */
12724 dd->num_rcv_contexts = total_contexts;
12725 dd->n_krcv_queues = num_kernel_contexts;
12726 dd->first_user_ctxt = num_kernel_contexts;
affa48de 12727 dd->num_user_contexts = num_user_contexts;
77241056
MM		 12728 dd->freectxts = num_user_contexts;
12729 dd_dev_info(dd,
17fb4f29
JJ		 12730 "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
12731 (int)dd->chip_rcv_contexts,
12732 (int)dd->num_rcv_contexts,
12733 (int)dd->n_krcv_queues,
12734 (int)dd->num_rcv_contexts - dd->n_krcv_queues);
77241056
MM		 12735
12736 /*
12737 * Receive array allocation:
12738 * All RcvArray entries are divided into groups of 8. This
12739 * is required by the hardware and will speed up writes to
12740 * consecutive entries by using write-combining of the entire
12741 * cacheline.
12742 *
12743 * The number of groups are evenly divided among all contexts.
12744 * any left over groups will be given to the first N user
12745 * contexts.
12746 */
12747 dd->rcv_entries.group_size = RCV_INCREMENT;
12748 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
12749 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
12750 dd->rcv_entries.nctxt_extra = ngroups -
12751 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
12752 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
12753 dd->rcv_entries.ngroups,
12754 dd->rcv_entries.nctxt_extra);
12755 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
12756 MAX_EAGER_ENTRIES * 2) {
12757 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
12758 dd->rcv_entries.group_size;
12759 dd_dev_info(dd,
17fb4f29
JJ		 12760 "RcvArray group count too high, change to %u\n",
12761 dd->rcv_entries.ngroups);
77241056
MM		 12762 dd->rcv_entries.nctxt_extra = 0;
12763 }
12764 /*
12765 * PIO send contexts
12766 */
12767 ret = init_sc_pools_and_sizes(dd);
12768 if (ret >= 0) { /* success */
12769 dd->num_send_contexts = ret;
12770 dd_dev_info(
12771 dd,
12772 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d)\n",
12773 dd->chip_send_contexts,
12774 dd->num_send_contexts,
12775 dd->sc_sizes[SC_KERNEL].count,
12776 dd->sc_sizes[SC_ACK].count,
12777 dd->sc_sizes[SC_USER].count);
12778 ret = 0; /* success */
12779 }
12780
12781 return ret;
12782}
12783
12784/*
12785 * Set the device/port partition key table. The MAD code
12786 * will ensure that, at least, the partial management
12787 * partition key is present in the table.
12788 */
12789static void set_partition_keys(struct hfi1_pportdata *ppd)
12790{
12791 struct hfi1_devdata *dd = ppd->dd;
12792 u64 reg = 0;
12793 int i;
12794
12795 dd_dev_info(dd, "Setting partition keys\n");
12796 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
12797 reg |= (ppd->pkeys[i] &
12798 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
12799 ((i % 4) *
12800 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
12801 /* Each register holds 4 PKey values. */
12802 if ((i % 4) == 3) {
12803 write_csr(dd, RCV_PARTITION_KEY +
12804 ((i - 3) * 2), reg);
12805 reg = 0;
12806 }
12807 }
12808
12809 /* Always enable HW pkeys check when pkeys table is set */
12810 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
12811}
12812
12813/*
12814 * These CSRs and memories are uninitialized on reset and must be
12815 * written before reading to set the ECC/parity bits.
12816 *
12817 * NOTE: All user context CSRs that are not mmaped write-only
12818 * (e.g. the TID flows) must be initialized even if the driver never
12819 * reads them.
12820 */
12821static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
12822{
12823 int i, j;
12824
12825 /* CceIntMap */
12826 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
8638b77f 12827 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
77241056
MM		 12828
12829 /* SendCtxtCreditReturnAddr */
12830 for (i = 0; i < dd->chip_send_contexts; i++)
12831 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
12832
12833 /* PIO Send buffers */
12834 /* SDMA Send buffers */
4d114fdd
JJ		 12835 /*
12836 * These are not normally read, and (presently) have no method
12837 * to be read, so are not pre-initialized
12838 */
77241056
MM		 12839
12840 /* RcvHdrAddr */
12841 /* RcvHdrTailAddr */
12842 /* RcvTidFlowTable */
12843 for (i = 0; i < dd->chip_rcv_contexts; i++) {
12844 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
12845 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
12846 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
8638b77f 12847 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
77241056
MM		 12848 }
12849
12850 /* RcvArray */
12851 for (i = 0; i < dd->chip_rcv_array_count; i++)
8638b77f 12852 write_csr(dd, RCV_ARRAY + (8 * i),
17fb4f29 12853 RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
77241056
MM		 12854
12855 /* RcvQPMapTable */
12856 for (i = 0; i < 32; i++)
12857 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
12858}
12859
12860/*
12861 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
12862 */
12863static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
12864 u64 ctrl_bits)
12865{
12866 unsigned long timeout;
12867 u64 reg;
12868
12869 /* is the condition present? */
12870 reg = read_csr(dd, CCE_STATUS);
12871 if ((reg & status_bits) == 0)
12872 return;
12873
12874 /* clear the condition */
12875 write_csr(dd, CCE_CTRL, ctrl_bits);
12876
12877 /* wait for the condition to clear */
12878 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
12879 while (1) {
12880 reg = read_csr(dd, CCE_STATUS);
12881 if ((reg & status_bits) == 0)
12882 return;
12883 if (time_after(jiffies, timeout)) {
12884 dd_dev_err(dd,
17fb4f29
JJ		 12885 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
12886 status_bits, reg & status_bits);
77241056
MM		 12887 return;
12888 }
12889 udelay(1);
12890 }
12891}
12892
12893/* set CCE CSRs to chip reset defaults */
12894static void reset_cce_csrs(struct hfi1_devdata *dd)
12895{
12896 int i;
12897
12898 /* CCE_REVISION read-only */
12899 /* CCE_REVISION2 read-only */
12900 /* CCE_CTRL - bits clear automatically */
12901 /* CCE_STATUS read-only, use CceCtrl to clear */
12902 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
12903 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
12904 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
12905 for (i = 0; i < CCE_NUM_SCRATCH; i++)
12906 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
12907 /* CCE_ERR_STATUS read-only */
12908 write_csr(dd, CCE_ERR_MASK, 0);
12909 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
12910 /* CCE_ERR_FORCE leave alone */
12911 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
12912 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
12913 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
12914 /* CCE_PCIE_CTRL leave alone */
12915 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
12916 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
12917 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
17fb4f29 12918 CCE_MSIX_TABLE_UPPER_RESETCSR);
77241056
MM		 12919 }
12920 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
12921 /* CCE_MSIX_PBA read-only */
12922 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
12923 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
12924 }
12925 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
12926 write_csr(dd, CCE_INT_MAP, 0);
12927 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
12928 /* CCE_INT_STATUS read-only */
12929 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
12930 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
12931 /* CCE_INT_FORCE leave alone */
12932 /* CCE_INT_BLOCKED read-only */
12933 }
12934 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
12935 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
12936}
12937
12938/* set ASIC CSRs to chip reset defaults */
12939static void reset_asic_csrs(struct hfi1_devdata *dd)
12940{
77241056
MM		 12941 int i;
12942
12943 /*
12944 * If the HFIs are shared between separate nodes or VMs,
12945 * then more will need to be done here. One idea is a module
12946 * parameter that returns early, letting the first power-on or
12947 * a known first load do the reset and blocking all others.
12948 */
12949
7c03ed85
EH		 12950 if (!(dd->flags & HFI1_DO_INIT_ASIC))
12951 return;
77241056
MM		 12952
12953 if (dd->icode != ICODE_FPGA_EMULATION) {
12954 /* emulation does not have an SBus - leave these alone */
12955 /*
12956 * All writes to ASIC_CFG_SBUS_REQUEST do something.
12957 * Notes:
12958 * o The reset is not zero if aimed at the core. See the
12959 * SBus documentation for details.
12960 * o If the SBus firmware has been updated (e.g. by the BIOS),
12961 * will the reset revert that?
12962 */
12963 /* ASIC_CFG_SBUS_REQUEST leave alone */
12964 write_csr(dd, ASIC_CFG_SBUS_EXECUTE, 0);
12965 }
12966 /* ASIC_SBUS_RESULT read-only */
12967 write_csr(dd, ASIC_STS_SBUS_COUNTERS, 0);
12968 for (i = 0; i < ASIC_NUM_SCRATCH; i++)
12969 write_csr(dd, ASIC_CFG_SCRATCH + (8 * i), 0);
12970 write_csr(dd, ASIC_CFG_MUTEX, 0); /* this will clear it */
7c03ed85
EH		 12971
12972 /* We might want to retain this state across FLR if we ever use it */
77241056 12973 write_csr(dd, ASIC_CFG_DRV_STR, 0);
7c03ed85 12974
4ef98989 12975 /* ASIC_CFG_THERM_POLL_EN leave alone */
77241056
MM		 12976 /* ASIC_STS_THERM read-only */
12977 /* ASIC_CFG_RESET leave alone */
12978
12979 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, 0);
12980 /* ASIC_PCIE_SD_HOST_STATUS read-only */
12981 write_csr(dd, ASIC_PCIE_SD_INTRPT_DATA_CODE, 0);
12982 write_csr(dd, ASIC_PCIE_SD_INTRPT_ENABLE, 0);
12983 /* ASIC_PCIE_SD_INTRPT_PROGRESS read-only */
12984 write_csr(dd, ASIC_PCIE_SD_INTRPT_STATUS, ~0ull); /* clear */
12985 /* ASIC_HFI0_PCIE_SD_INTRPT_RSPD_DATA read-only */
12986 /* ASIC_HFI1_PCIE_SD_INTRPT_RSPD_DATA read-only */
12987 for (i = 0; i < 16; i++)
12988 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (8 * i), 0);
12989
12990 /* ASIC_GPIO_IN read-only */
12991 write_csr(dd, ASIC_GPIO_OE, 0);
12992 write_csr(dd, ASIC_GPIO_INVERT, 0);
12993 write_csr(dd, ASIC_GPIO_OUT, 0);
12994 write_csr(dd, ASIC_GPIO_MASK, 0);
12995 /* ASIC_GPIO_STATUS read-only */
12996 write_csr(dd, ASIC_GPIO_CLEAR, ~0ull);
12997 /* ASIC_GPIO_FORCE leave alone */
12998
12999 /* ASIC_QSFP1_IN read-only */
13000 write_csr(dd, ASIC_QSFP1_OE, 0);
13001 write_csr(dd, ASIC_QSFP1_INVERT, 0);
13002 write_csr(dd, ASIC_QSFP1_OUT, 0);
13003 write_csr(dd, ASIC_QSFP1_MASK, 0);
13004 /* ASIC_QSFP1_STATUS read-only */
13005 write_csr(dd, ASIC_QSFP1_CLEAR, ~0ull);
13006 /* ASIC_QSFP1_FORCE leave alone */
13007
13008 /* ASIC_QSFP2_IN read-only */
13009 write_csr(dd, ASIC_QSFP2_OE, 0);
13010 write_csr(dd, ASIC_QSFP2_INVERT, 0);
13011 write_csr(dd, ASIC_QSFP2_OUT, 0);
13012 write_csr(dd, ASIC_QSFP2_MASK, 0);
13013 /* ASIC_QSFP2_STATUS read-only */
13014 write_csr(dd, ASIC_QSFP2_CLEAR, ~0ull);
13015 /* ASIC_QSFP2_FORCE leave alone */
13016
13017 write_csr(dd, ASIC_EEP_CTL_STAT, ASIC_EEP_CTL_STAT_RESETCSR);
13018 /* this also writes a NOP command, clearing paging mode */
13019 write_csr(dd, ASIC_EEP_ADDR_CMD, 0);
13020 write_csr(dd, ASIC_EEP_DATA, 0);
77241056
MM		 13021}
13022
13023/* set MISC CSRs to chip reset defaults */
13024static void reset_misc_csrs(struct hfi1_devdata *dd)
13025{
13026 int i;
13027
13028 for (i = 0; i < 32; i++) {
13029 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13030 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13031 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13032 }
4d114fdd
JJ		 13033 /*
13034 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13035 * only be written 128-byte chunks
13036 */
77241056
MM		 13037 /* init RSA engine to clear lingering errors */
13038 write_csr(dd, MISC_CFG_RSA_CMD, 1);
13039 write_csr(dd, MISC_CFG_RSA_MU, 0);
13040 write_csr(dd, MISC_CFG_FW_CTRL, 0);
13041 /* MISC_STS_8051_DIGEST read-only */
13042 /* MISC_STS_SBM_DIGEST read-only */
13043 /* MISC_STS_PCIE_DIGEST read-only */
13044 /* MISC_STS_FAB_DIGEST read-only */
13045 /* MISC_ERR_STATUS read-only */
13046 write_csr(dd, MISC_ERR_MASK, 0);
13047 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13048 /* MISC_ERR_FORCE leave alone */
13049}
13050
13051/* set TXE CSRs to chip reset defaults */
13052static void reset_txe_csrs(struct hfi1_devdata *dd)
13053{
13054 int i;
13055
13056 /*
13057 * TXE Kernel CSRs
13058 */
13059 write_csr(dd, SEND_CTRL, 0);
13060 __cm_reset(dd, 0); /* reset CM internal state */
13061 /* SEND_CONTEXTS read-only */
13062 /* SEND_DMA_ENGINES read-only */
13063 /* SEND_PIO_MEM_SIZE read-only */
13064 /* SEND_DMA_MEM_SIZE read-only */
13065 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13066 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
13067 /* SEND_PIO_ERR_STATUS read-only */
13068 write_csr(dd, SEND_PIO_ERR_MASK, 0);
13069 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13070 /* SEND_PIO_ERR_FORCE leave alone */
13071 /* SEND_DMA_ERR_STATUS read-only */
13072 write_csr(dd, SEND_DMA_ERR_MASK, 0);
13073 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13074 /* SEND_DMA_ERR_FORCE leave alone */
13075 /* SEND_EGRESS_ERR_STATUS read-only */
13076 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13077 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13078 /* SEND_EGRESS_ERR_FORCE leave alone */
13079 write_csr(dd, SEND_BTH_QP, 0);
13080 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13081 write_csr(dd, SEND_SC2VLT0, 0);
13082 write_csr(dd, SEND_SC2VLT1, 0);
13083 write_csr(dd, SEND_SC2VLT2, 0);
13084 write_csr(dd, SEND_SC2VLT3, 0);
13085 write_csr(dd, SEND_LEN_CHECK0, 0);
13086 write_csr(dd, SEND_LEN_CHECK1, 0);
13087 /* SEND_ERR_STATUS read-only */
13088 write_csr(dd, SEND_ERR_MASK, 0);
13089 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13090 /* SEND_ERR_FORCE read-only */
13091 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
8638b77f 13092 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
77241056 13093 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
8638b77f
JJ		 13094 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13095 for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++)
13096 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
77241056 13097 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
8638b77f 13098 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
77241056 13099 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
8638b77f 13100 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
77241056 13101 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
17fb4f29 13102 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
77241056
MM		 13103 /* SEND_CM_CREDIT_USED_STATUS read-only */
13104 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13105 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13106 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13107 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13108 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13109 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 13110 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
77241056
MM		 13111 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13112 /* SEND_CM_CREDIT_USED_VL read-only */
13113 /* SEND_CM_CREDIT_USED_VL15 read-only */
13114 /* SEND_EGRESS_CTXT_STATUS read-only */
13115 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13116 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13117 /* SEND_EGRESS_ERR_INFO read-only */
13118 /* SEND_EGRESS_ERR_SOURCE read-only */
13119
13120 /*
13121 * TXE Per-Context CSRs
13122 */
13123 for (i = 0; i < dd->chip_send_contexts; i++) {
13124 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13125 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13126 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13127 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13128 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13129 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13130 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13131 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13132 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13133 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13134 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13135 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13136 }
13137
13138 /*
13139 * TXE Per-SDMA CSRs
13140 */
13141 for (i = 0; i < dd->chip_sdma_engines; i++) {
13142 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13143 /* SEND_DMA_STATUS read-only */
13144 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13145 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13146 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13147 /* SEND_DMA_HEAD read-only */
13148 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13149 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13150 /* SEND_DMA_IDLE_CNT read-only */
13151 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13152 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13153 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13154 /* SEND_DMA_ENG_ERR_STATUS read-only */
13155 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13156 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13157 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13158 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13159 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13160 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13161 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13162 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13163 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13164 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13165 }
13166}
13167
13168/*
13169 * Expect on entry:
13170 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13171 */
13172static void init_rbufs(struct hfi1_devdata *dd)
13173{
13174 u64 reg;
13175 int count;
13176
13177 /*
13178 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13179 * clear.
13180 */
13181 count = 0;
13182 while (1) {
13183 reg = read_csr(dd, RCV_STATUS);
13184 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13185 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13186 break;
13187 /*
13188 * Give up after 1ms - maximum wait time.
13189 *
13190 * RBuf size is 148KiB. Slowest possible is PCIe Gen1 x1 at
13191 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
13192 * 148 KB / (66% * 250MB/s) = 920us
13193 */
13194 if (count++ > 500) {
13195 dd_dev_err(dd,
17fb4f29
JJ		 13196 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13197 __func__, reg);
77241056
MM		 13198 break;
13199 }
13200 udelay(2); /* do not busy-wait the CSR */
13201 }
13202
13203 /* start the init - expect RcvCtrl to be 0 */
13204 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13205
13206 /*
13207 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
13208 * period after the write before RcvStatus.RxRbufInitDone is valid.
13209 * The delay in the first run through the loop below is sufficient and
13210 * required before the first read of RcvStatus.RxRbufInintDone.
13211 */
13212 read_csr(dd, RCV_CTRL);
13213
13214 /* wait for the init to finish */
13215 count = 0;
13216 while (1) {
13217 /* delay is required first time through - see above */
13218 udelay(2); /* do not busy-wait the CSR */
13219 reg = read_csr(dd, RCV_STATUS);
13220 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13221 break;
13222
13223 /* give up after 100us - slowest possible at 33MHz is 73us */
13224 if (count++ > 50) {
13225 dd_dev_err(dd,
17fb4f29
JJ		 13226 "%s: RcvStatus.RxRbufInit not set, continuing\n",
13227 __func__);
77241056
MM		 13228 break;
13229 }
13230 }
13231}
13232
13233/* set RXE CSRs to chip reset defaults */
13234static void reset_rxe_csrs(struct hfi1_devdata *dd)
13235{
13236 int i, j;
13237
13238 /*
13239 * RXE Kernel CSRs
13240 */
13241 write_csr(dd, RCV_CTRL, 0);
13242 init_rbufs(dd);
13243 /* RCV_STATUS read-only */
13244 /* RCV_CONTEXTS read-only */
13245 /* RCV_ARRAY_CNT read-only */
13246 /* RCV_BUF_SIZE read-only */
13247 write_csr(dd, RCV_BTH_QP, 0);
13248 write_csr(dd, RCV_MULTICAST, 0);
13249 write_csr(dd, RCV_BYPASS, 0);
13250 write_csr(dd, RCV_VL15, 0);
13251 /* this is a clear-down */
13252 write_csr(dd, RCV_ERR_INFO,
17fb4f29 13253 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
77241056
MM		 13254 /* RCV_ERR_STATUS read-only */
13255 write_csr(dd, RCV_ERR_MASK, 0);
13256 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13257 /* RCV_ERR_FORCE leave alone */
13258 for (i = 0; i < 32; i++)
13259 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13260 for (i = 0; i < 4; i++)
13261 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13262 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13263 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13264 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13265 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13266 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
13267 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
13268 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
13269 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
13270 }
13271 for (i = 0; i < 32; i++)
13272 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13273
13274 /*
13275 * RXE Kernel and User Per-Context CSRs
13276 */
13277 for (i = 0; i < dd->chip_rcv_contexts; i++) {
13278 /* kernel */
13279 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13280 /* RCV_CTXT_STATUS read-only */
13281 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13282 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13283 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13284 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13285 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13286 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13287 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13288 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13289 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13290 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13291
13292 /* user */
13293 /* RCV_HDR_TAIL read-only */
13294 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13295 /* RCV_EGR_INDEX_TAIL read-only */
13296 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13297 /* RCV_EGR_OFFSET_TAIL read-only */
13298 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
17fb4f29
JJ		 13299 write_uctxt_csr(dd, i,
13300 RCV_TID_FLOW_TABLE + (8 * j), 0);
77241056
MM		 13301 }
13302 }
13303}
13304
13305/*
13306 * Set sc2vl tables.
13307 *
13308 * They power on to zeros, so to avoid send context errors
13309 * they need to be set:
13310 *
13311 * SC 0-7 -> VL 0-7 (respectively)
13312 * SC 15 -> VL 15
13313 * otherwise
13314 * -> VL 0
13315 */
13316static void init_sc2vl_tables(struct hfi1_devdata *dd)
13317{
13318 int i;
13319 /* init per architecture spec, constrained by hardware capability */
13320
13321 /* HFI maps sent packets */
13322 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13323 0,
13324 0, 0, 1, 1,
13325 2, 2, 3, 3,
13326 4, 4, 5, 5,
13327 6, 6, 7, 7));
13328 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13329 1,
13330 8, 0, 9, 0,
13331 10, 0, 11, 0,
13332 12, 0, 13, 0,
13333 14, 0, 15, 15));
13334 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13335 2,
13336 16, 0, 17, 0,
13337 18, 0, 19, 0,
13338 20, 0, 21, 0,
13339 22, 0, 23, 0));
13340 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13341 3,
13342 24, 0, 25, 0,
13343 26, 0, 27, 0,
13344 28, 0, 29, 0,
13345 30, 0, 31, 0));
13346
13347 /* DC maps received packets */
13348 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13349 15_0,
13350 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
13351 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13352 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13353 31_16,
13354 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13355 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13356
13357 /* initialize the cached sc2vl values consistently with h/w */
13358 for (i = 0; i < 32; i++) {
13359 if (i < 8 || i == 15)
13360 *((u8 *)(dd->sc2vl) + i) = (u8)i;
13361 else
13362 *((u8 *)(dd->sc2vl) + i) = 0;
13363 }
13364}
13365
13366/*
13367 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
13368 * depend on the chip going through a power-on reset - a driver may be loaded
13369 * and unloaded many times.
13370 *
13371 * Do not write any CSR values to the chip in this routine - there may be
13372 * a reset following the (possible) FLR in this routine.
13373 *
13374 */
13375static void init_chip(struct hfi1_devdata *dd)
13376{
13377 int i;
13378
13379 /*
13380 * Put the HFI CSRs in a known state.
13381 * Combine this with a DC reset.
13382 *
13383 * Stop the device from doing anything while we do a
13384 * reset. We know there are no other active users of
13385 * the device since we are now in charge. Turn off
13386 * off all outbound and inbound traffic and make sure
13387 * the device does not generate any interrupts.
13388 */
13389
13390 /* disable send contexts and SDMA engines */
13391 write_csr(dd, SEND_CTRL, 0);
13392 for (i = 0; i < dd->chip_send_contexts; i++)
13393 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13394 for (i = 0; i < dd->chip_sdma_engines; i++)
13395 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13396 /* disable port (turn off RXE inbound traffic) and contexts */
13397 write_csr(dd, RCV_CTRL, 0);
13398 for (i = 0; i < dd->chip_rcv_contexts; i++)
13399 write_csr(dd, RCV_CTXT_CTRL, 0);
13400 /* mask all interrupt sources */
13401 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 13402 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
77241056
MM		 13403
13404 /*
13405 * DC Reset: do a full DC reset before the register clear.
13406 * A recommended length of time to hold is one CSR read,
13407 * so reread the CceDcCtrl. Then, hold the DC in reset
13408 * across the clear.
13409 */
13410 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
50e5dcbe 13411 (void)read_csr(dd, CCE_DC_CTRL);
77241056
MM		 13412
13413 if (use_flr) {
13414 /*
13415 * A FLR will reset the SPC core and part of the PCIe.
13416 * The parts that need to be restored have already been
13417 * saved.
13418 */
13419 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13420
13421 /* do the FLR, the DC reset will remain */
13422 hfi1_pcie_flr(dd);
13423
13424 /* restore command and BARs */
13425 restore_pci_variables(dd);
13426
995deafa 13427 if (is_ax(dd)) {
77241056
MM		 13428 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13429 hfi1_pcie_flr(dd);
13430 restore_pci_variables(dd);
13431 }
13432
7c03ed85 13433 reset_asic_csrs(dd);
77241056
MM		 13434 } else {
13435 dd_dev_info(dd, "Resetting CSRs with writes\n");
13436 reset_cce_csrs(dd);
13437 reset_txe_csrs(dd);
13438 reset_rxe_csrs(dd);
13439 reset_asic_csrs(dd);
13440 reset_misc_csrs(dd);
13441 }
13442 /* clear the DC reset */
13443 write_csr(dd, CCE_DC_CTRL, 0);
7c03ed85 13444
77241056 13445 /* Set the LED off */
773d0451
SS		 13446 setextled(dd, 0);
13447
77241056
MM		 13448 /*
13449 * Clear the QSFP reset.
72a67ba2 13450 * An FLR enforces a 0 on all out pins. The driver does not touch
77241056 13451 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
72a67ba2 13452 * anything plugged constantly in reset, if it pays attention
77241056 13453 * to RESET_N.
72a67ba2 13454 * Prime examples of this are optical cables. Set all pins high.
77241056
MM		 13455 * I2CCLK and I2CDAT will change per direction, and INT_N and
13456 * MODPRS_N are input only and their value is ignored.
13457 */
72a67ba2
EH		 13458 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13459 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
77241056
MM		 13460}
13461
13462static void init_early_variables(struct hfi1_devdata *dd)
13463{
13464 int i;
13465
13466 /* assign link credit variables */
13467 dd->vau = CM_VAU;
13468 dd->link_credits = CM_GLOBAL_CREDITS;
995deafa 13469 if (is_ax(dd))
77241056
MM		 13470 dd->link_credits--;
13471 dd->vcu = cu_to_vcu(hfi1_cu);
13472 /* enough room for 8 MAD packets plus header - 17K */
13473 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
13474 if (dd->vl15_init > dd->link_credits)
13475 dd->vl15_init = dd->link_credits;
13476
13477 write_uninitialized_csrs_and_memories(dd);
13478
13479 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
13480 for (i = 0; i < dd->num_pports; i++) {
13481 struct hfi1_pportdata *ppd = &dd->pport[i];
13482
13483 set_partition_keys(ppd);
13484 }
13485 init_sc2vl_tables(dd);
13486}
13487
13488static void init_kdeth_qp(struct hfi1_devdata *dd)
13489{
13490 /* user changed the KDETH_QP */
13491 if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
13492 /* out of range or illegal value */
13493 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
13494 kdeth_qp = 0;
13495 }
13496 if (kdeth_qp == 0) /* not set, or failed range check */
13497 kdeth_qp = DEFAULT_KDETH_QP;
13498
13499 write_csr(dd, SEND_BTH_QP,
17fb4f29
JJ		 13500 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
13501 SEND_BTH_QP_KDETH_QP_SHIFT);
77241056
MM		 13502
13503 write_csr(dd, RCV_BTH_QP,
17fb4f29
JJ		 13504 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
13505 RCV_BTH_QP_KDETH_QP_SHIFT);
77241056
MM		 13506}
13507
13508/**
13509 * init_qpmap_table
13510 * @dd - device data
13511 * @first_ctxt - first context
13512 * @last_ctxt - first context
13513 *
13514 * This return sets the qpn mapping table that
13515 * is indexed by qpn[8:1].
13516 *
13517 * The routine will round robin the 256 settings
13518 * from first_ctxt to last_ctxt.
13519 *
13520 * The first/last looks ahead to having specialized
13521 * receive contexts for mgmt and bypass. Normal
13522 * verbs traffic will assumed to be on a range
13523 * of receive contexts.
13524 */
13525static void init_qpmap_table(struct hfi1_devdata *dd,
13526 u32 first_ctxt,
13527 u32 last_ctxt)
13528{
13529 u64 reg = 0;
13530 u64 regno = RCV_QP_MAP_TABLE;
13531 int i;
13532 u64 ctxt = first_ctxt;
13533
13534 for (i = 0; i < 256;) {
77241056
MM		 13535 reg |= ctxt << (8 * (i % 8));
13536 i++;
13537 ctxt++;
13538 if (ctxt > last_ctxt)
13539 ctxt = first_ctxt;
13540 if (i % 8 == 0) {
13541 write_csr(dd, regno, reg);
13542 reg = 0;
13543 regno += 8;
13544 }
13545 }
13546 if (i % 8)
13547 write_csr(dd, regno, reg);
13548
13549 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
13550 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
13551}
13552
13553/**
13554 * init_qos - init RX qos
13555 * @dd - device data
13556 * @first_context
13557 *
13558 * This routine initializes Rule 0 and the
13559 * RSM map table to implement qos.
13560 *
13561 * If all of the limit tests succeed,
13562 * qos is applied based on the array
13563 * interpretation of krcvqs where
13564 * entry 0 is VL0.
13565 *
13566 * The number of vl bits (n) and the number of qpn
13567 * bits (m) are computed to feed both the RSM map table
13568 * and the single rule.
13569 *
13570 */
13571static void init_qos(struct hfi1_devdata *dd, u32 first_ctxt)
13572{
13573 u8 max_by_vl = 0;
13574 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
13575 u64 *rsmmap;
13576 u64 reg;
995deafa 13577 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
77241056
MM		 13578
13579 /* validate */
13580 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
13581 num_vls == 1 ||
13582 krcvqsset <= 1)
13583 goto bail;
13584 for (i = 0; i < min_t(unsigned, num_vls, krcvqsset); i++)
13585 if (krcvqs[i] > max_by_vl)
13586 max_by_vl = krcvqs[i];
13587 if (max_by_vl > 32)
13588 goto bail;
13589 qpns_per_vl = __roundup_pow_of_two(max_by_vl);
13590 /* determine bits vl */
13591 n = ilog2(num_vls);
13592 /* determine bits for qpn */
13593 m = ilog2(qpns_per_vl);
13594 if ((m + n) > 7)
13595 goto bail;
13596 if (num_vls * qpns_per_vl > dd->chip_rcv_contexts)
13597 goto bail;
13598 rsmmap = kmalloc_array(NUM_MAP_REGS, sizeof(u64), GFP_KERNEL);
859bcad9
EH		 13599 if (!rsmmap)
13600 goto bail;
77241056
MM		 13601 memset(rsmmap, rxcontext, NUM_MAP_REGS * sizeof(u64));
13602 /* init the local copy of the table */
13603 for (i = 0, ctxt = first_ctxt; i < num_vls; i++) {
13604 unsigned tctxt;
13605
13606 for (qpn = 0, tctxt = ctxt;
13607 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
13608 unsigned idx, regoff, regidx;
13609
13610 /* generate index <= 128 */
13611 idx = (qpn << n) ^ i;
13612 regoff = (idx % 8) * 8;
13613 regidx = idx / 8;
13614 reg = rsmmap[regidx];
13615 /* replace 0xff with context number */
13616 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
13617 << regoff);
13618 reg |= (u64)(tctxt++) << regoff;
13619 rsmmap[regidx] = reg;
13620 if (tctxt == ctxt + krcvqs[i])
13621 tctxt = ctxt;
13622 }
13623 ctxt += krcvqs[i];
13624 }
13625 /* flush cached copies to chip */
13626 for (i = 0; i < NUM_MAP_REGS; i++)
13627 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rsmmap[i]);
13628 /* add rule0 */
13629 write_csr(dd, RCV_RSM_CFG /* + (8 * 0) */,
17fb4f29
JJ		 13630 RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_MASK <<
13631 RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_SHIFT |
13632 2ull << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
77241056 13633 write_csr(dd, RCV_RSM_SELECT /* + (8 * 0) */,
17fb4f29
JJ		 13634 LRH_BTH_MATCH_OFFSET << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
13635 LRH_SC_MATCH_OFFSET << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
13636 LRH_SC_SELECT_OFFSET << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
13637 ((u64)n) << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
13638 QPN_SELECT_OFFSET << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
13639 ((u64)m + (u64)n) << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
77241056 13640 write_csr(dd, RCV_RSM_MATCH /* + (8 * 0) */,
17fb4f29
JJ		 13641 LRH_BTH_MASK << RCV_RSM_MATCH_MASK1_SHIFT |
13642 LRH_BTH_VALUE << RCV_RSM_MATCH_VALUE1_SHIFT |
13643 LRH_SC_MASK << RCV_RSM_MATCH_MASK2_SHIFT |
13644 LRH_SC_VALUE << RCV_RSM_MATCH_VALUE2_SHIFT);
77241056
MM		 13645 /* Enable RSM */
13646 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
13647 kfree(rsmmap);
82c2611d
NV		 13648 /* map everything else to first context */
13649 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, MIN_KERNEL_KCTXTS - 1);
77241056
MM		 13650 dd->qos_shift = n + 1;
13651 return;
13652bail:
13653 dd->qos_shift = 1;
82c2611d 13654 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
77241056
MM		 13655}
13656
13657static void init_rxe(struct hfi1_devdata *dd)
13658{
13659 /* enable all receive errors */
13660 write_csr(dd, RCV_ERR_MASK, ~0ull);
13661 /* setup QPN map table - start where VL15 context leaves off */
17fb4f29
JJ		 13662 init_qos(dd, dd->n_krcv_queues > MIN_KERNEL_KCTXTS ?
13663 MIN_KERNEL_KCTXTS : 0);
77241056
MM		 13664 /*
13665 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
13666 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
13667 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
13668 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
13669 * Max_PayLoad_Size set to its minimum of 128.
13670 *
13671 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
13672 * (64 bytes). Max_Payload_Size is possibly modified upward in
13673 * tune_pcie_caps() which is called after this routine.
13674 */
13675}
13676
13677static void init_other(struct hfi1_devdata *dd)
13678{
13679 /* enable all CCE errors */
13680 write_csr(dd, CCE_ERR_MASK, ~0ull);
13681 /* enable *some* Misc errors */
13682 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
13683 /* enable all DC errors, except LCB */
13684 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
13685 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
13686}
13687
13688/*
13689 * Fill out the given AU table using the given CU. A CU is defined in terms
13690 * AUs. The table is a an encoding: given the index, how many AUs does that
13691 * represent?
13692 *
13693 * NOTE: Assumes that the register layout is the same for the
13694 * local and remote tables.
13695 */
13696static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
13697 u32 csr0to3, u32 csr4to7)
13698{
13699 write_csr(dd, csr0to3,
17fb4f29
JJ		 13700 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
13701 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
13702 2ull * cu <<
13703 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
13704 4ull * cu <<
13705 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
77241056 13706 write_csr(dd, csr4to7,
17fb4f29
JJ		 13707 8ull * cu <<
13708 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
13709 16ull * cu <<
13710 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
13711 32ull * cu <<
13712 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
13713 64ull * cu <<
13714 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
77241056
MM		 13715}
13716
13717static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13718{
13719 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
17fb4f29 13720 SEND_CM_LOCAL_AU_TABLE4_TO7);
77241056
MM		 13721}
13722
13723void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13724{
13725 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
17fb4f29 13726 SEND_CM_REMOTE_AU_TABLE4_TO7);
77241056
MM		 13727}
13728
13729static void init_txe(struct hfi1_devdata *dd)
13730{
13731 int i;
13732
13733 /* enable all PIO, SDMA, general, and Egress errors */
13734 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
13735 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
13736 write_csr(dd, SEND_ERR_MASK, ~0ull);
13737 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
13738
13739 /* enable all per-context and per-SDMA engine errors */
13740 for (i = 0; i < dd->chip_send_contexts; i++)
13741 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
13742 for (i = 0; i < dd->chip_sdma_engines; i++)
13743 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
13744
13745 /* set the local CU to AU mapping */
13746 assign_local_cm_au_table(dd, dd->vcu);
13747
13748 /*
13749 * Set reasonable default for Credit Return Timer
13750 * Don't set on Simulator - causes it to choke.
13751 */
13752 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
13753 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
13754}
13755
13756int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
13757{
13758 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
13759 unsigned sctxt;
13760 int ret = 0;
13761 u64 reg;
13762
13763 if (!rcd || !rcd->sc) {
13764 ret = -EINVAL;
13765 goto done;
13766 }
13767 sctxt = rcd->sc->hw_context;
13768 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
13769 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
13770 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
13771 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
13772 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
13773 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
13774 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
13775 /*
13776 * Enable send-side J_KEY integrity check, unless this is A0 h/w
77241056 13777 */
995deafa 13778 if (!is_ax(dd)) {
77241056
MM		 13779 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13780 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
13781 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13782 }
13783
13784 /* Enable J_KEY check on receive context. */
13785 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
13786 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
13787 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
13788 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
13789done:
13790 return ret;
13791}
13792
13793int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
13794{
13795 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
13796 unsigned sctxt;
13797 int ret = 0;
13798 u64 reg;
13799
13800 if (!rcd || !rcd->sc) {
13801 ret = -EINVAL;
13802 goto done;
13803 }
13804 sctxt = rcd->sc->hw_context;
13805 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
13806 /*
13807 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
13808 * This check would not have been enabled for A0 h/w, see
13809 * set_ctxt_jkey().
13810 */
995deafa 13811 if (!is_ax(dd)) {
77241056
MM		 13812 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13813 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
13814 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13815 }
13816 /* Turn off the J_KEY on the receive side */
13817 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
13818done:
13819 return ret;
13820}
13821
13822int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
13823{
13824 struct hfi1_ctxtdata *rcd;
13825 unsigned sctxt;
13826 int ret = 0;
13827 u64 reg;
13828
e490974e 13829 if (ctxt < dd->num_rcv_contexts) {
77241056 13830 rcd = dd->rcd[ctxt];
e490974e 13831 } else {
77241056
MM		 13832 ret = -EINVAL;
13833 goto done;
13834 }
13835 if (!rcd || !rcd->sc) {
13836 ret = -EINVAL;
13837 goto done;
13838 }
13839 sctxt = rcd->sc->hw_context;
13840 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
13841 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
13842 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
13843 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13844 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
13845 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13846done:
13847 return ret;
13848}
13849
13850int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
13851{
13852 struct hfi1_ctxtdata *rcd;
13853 unsigned sctxt;
13854 int ret = 0;
13855 u64 reg;
13856
e490974e 13857 if (ctxt < dd->num_rcv_contexts) {
77241056 13858 rcd = dd->rcd[ctxt];
e490974e 13859 } else {
77241056
MM		 13860 ret = -EINVAL;
13861 goto done;
13862 }
13863 if (!rcd || !rcd->sc) {
13864 ret = -EINVAL;
13865 goto done;
13866 }
13867 sctxt = rcd->sc->hw_context;
13868 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13869 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
13870 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13871 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13872done:
13873 return ret;
13874}
13875
13876/*
13877 * Start doing the clean up the the chip. Our clean up happens in multiple
13878 * stages and this is just the first.
13879 */
13880void hfi1_start_cleanup(struct hfi1_devdata *dd)
13881{
affa48de 13882 aspm_exit(dd);
77241056
MM		 13883 free_cntrs(dd);
13884 free_rcverr(dd);
13885 clean_up_interrupts(dd);
13886}
13887
13888#define HFI_BASE_GUID(dev) \
13889 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
13890
13891/*
13892 * Certain chip functions need to be initialized only once per asic
13893 * instead of per-device. This function finds the peer device and
13894 * checks whether that chip initialization needs to be done by this
13895 * device.
13896 */
13897static void asic_should_init(struct hfi1_devdata *dd)
13898{
13899 unsigned long flags;
13900 struct hfi1_devdata *tmp, *peer = NULL;
13901
13902 spin_lock_irqsave(&hfi1_devs_lock, flags);
13903 /* Find our peer device */
13904 list_for_each_entry(tmp, &hfi1_dev_list, list) {
13905 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
13906 dd->unit != tmp->unit) {
13907 peer = tmp;
13908 break;
13909 }
13910 }
13911
13912 /*
13913 * "Claim" the ASIC for initialization if it hasn't been
13914 " "claimed" yet.
13915 */
13916 if (!peer || !(peer->flags & HFI1_DO_INIT_ASIC))
13917 dd->flags |= HFI1_DO_INIT_ASIC;
13918 spin_unlock_irqrestore(&hfi1_devs_lock, flags);
13919}
13920
5d9157aa
DL		 13921/*
13922 * Set dd->boardname. Use a generic name if a name is not returned from
13923 * EFI variable space.
13924 *
13925 * Return 0 on success, -ENOMEM if space could not be allocated.
13926 */
13927static int obtain_boardname(struct hfi1_devdata *dd)
13928{
13929 /* generic board description */
13930 const char generic[] =
13931 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
13932 unsigned long size;
13933 int ret;
13934
13935 ret = read_hfi1_efi_var(dd, "description", &size,
13936 (void **)&dd->boardname);
13937 if (ret) {
845f876d 13938 dd_dev_info(dd, "Board description not found\n");
5d9157aa
DL		 13939 /* use generic description */
13940 dd->boardname = kstrdup(generic, GFP_KERNEL);
13941 if (!dd->boardname)
13942 return -ENOMEM;
13943 }
13944 return 0;
13945}
13946
77241056 13947/**
7c03ed85 13948 * Allocate and initialize the device structure for the hfi.
77241056
MM		 13949 * @dev: the pci_dev for hfi1_ib device
13950 * @ent: pci_device_id struct for this dev
13951 *
13952 * Also allocates, initializes, and returns the devdata struct for this
13953 * device instance
13954 *
13955 * This is global, and is called directly at init to set up the
13956 * chip-specific function pointers for later use.
13957 */
13958struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
13959 const struct pci_device_id *ent)
13960{
13961 struct hfi1_devdata *dd;
13962 struct hfi1_pportdata *ppd;
13963 u64 reg;
13964 int i, ret;
13965 static const char * const inames[] = { /* implementation names */
13966 "RTL silicon",
13967 "RTL VCS simulation",
13968 "RTL FPGA emulation",
13969 "Functional simulator"
13970 };
13971
17fb4f29
JJ		 13972 dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
13973 sizeof(struct hfi1_pportdata));
77241056
MM		 13974 if (IS_ERR(dd))
13975 goto bail;
13976 ppd = dd->pport;
13977 for (i = 0; i < dd->num_pports; i++, ppd++) {
13978 int vl;
13979 /* init common fields */
13980 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
13981 /* DC supports 4 link widths */
13982 ppd->link_width_supported =
13983 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
13984 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
13985 ppd->link_width_downgrade_supported =
13986 ppd->link_width_supported;
13987 /* start out enabling only 4X */
13988 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
13989 ppd->link_width_downgrade_enabled =
13990 ppd->link_width_downgrade_supported;
13991 /* link width active is 0 when link is down */
13992 /* link width downgrade active is 0 when link is down */
13993
d0d236ea
JJ		 13994 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
13995 num_vls > HFI1_MAX_VLS_SUPPORTED) {
77241056
MM		 13996 hfi1_early_err(&pdev->dev,
13997 "Invalid num_vls %u, using %u VLs\n",
13998 num_vls, HFI1_MAX_VLS_SUPPORTED);
13999 num_vls = HFI1_MAX_VLS_SUPPORTED;
14000 }
14001 ppd->vls_supported = num_vls;
14002 ppd->vls_operational = ppd->vls_supported;
8a4d3444 14003 ppd->actual_vls_operational = ppd->vls_supported;
77241056
MM		 14004 /* Set the default MTU. */
14005 for (vl = 0; vl < num_vls; vl++)
14006 dd->vld[vl].mtu = hfi1_max_mtu;
14007 dd->vld[15].mtu = MAX_MAD_PACKET;
14008 /*
14009 * Set the initial values to reasonable default, will be set
14010 * for real when link is up.
14011 */
14012 ppd->lstate = IB_PORT_DOWN;
14013 ppd->overrun_threshold = 0x4;
14014 ppd->phy_error_threshold = 0xf;
14015 ppd->port_crc_mode_enabled = link_crc_mask;
14016 /* initialize supported LTP CRC mode */
14017 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14018 /* initialize enabled LTP CRC mode */
14019 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14020 /* start in offline */
14021 ppd->host_link_state = HLS_DN_OFFLINE;
14022 init_vl_arb_caches(ppd);
f45c8dc8 14023 ppd->last_pstate = 0xff; /* invalid value */
77241056
MM		 14024 }
14025
14026 dd->link_default = HLS_DN_POLL;
14027
14028 /*
14029 * Do remaining PCIe setup and save PCIe values in dd.
14030 * Any error printing is already done by the init code.
14031 * On return, we have the chip mapped.
14032 */
14033 ret = hfi1_pcie_ddinit(dd, pdev, ent);
14034 if (ret < 0)
14035 goto bail_free;
14036
14037 /* verify that reads actually work, save revision for reset check */
14038 dd->revision = read_csr(dd, CCE_REVISION);
14039 if (dd->revision == ~(u64)0) {
14040 dd_dev_err(dd, "cannot read chip CSRs\n");
14041 ret = -EINVAL;
14042 goto bail_cleanup;
14043 }
14044 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14045 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14046 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14047 & CCE_REVISION_CHIP_REV_MINOR_MASK;
14048
4d114fdd
JJ		 14049 /*
14050 * obtain the hardware ID - NOT related to unit, which is a
14051 * software enumeration
14052 */
77241056
MM		 14053 reg = read_csr(dd, CCE_REVISION2);
14054 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14055 & CCE_REVISION2_HFI_ID_MASK;
14056 /* the variable size will remove unwanted bits */
14057 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14058 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14059 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
17fb4f29
JJ		 14060 dd->icode < ARRAY_SIZE(inames) ?
14061 inames[dd->icode] : "unknown", (int)dd->irev);
77241056
MM		 14062
14063 /* speeds the hardware can support */
14064 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14065 /* speeds allowed to run at */
14066 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14067 /* give a reasonable active value, will be set on link up */
14068 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14069
14070 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
14071 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
14072 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
14073 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
14074 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
14075 /* fix up link widths for emulation _p */
14076 ppd = dd->pport;
14077 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14078 ppd->link_width_supported =
14079 ppd->link_width_enabled =
14080 ppd->link_width_downgrade_supported =
14081 ppd->link_width_downgrade_enabled =
14082 OPA_LINK_WIDTH_1X;
14083 }
14084 /* insure num_vls isn't larger than number of sdma engines */
14085 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
14086 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
11a5909b
DL		 14087 num_vls, dd->chip_sdma_engines);
14088 num_vls = dd->chip_sdma_engines;
14089 ppd->vls_supported = dd->chip_sdma_engines;
8a4d3444 14090 ppd->vls_operational = ppd->vls_supported;
77241056
MM		 14091 }
14092
14093 /*
14094 * Convert the ns parameter to the 64 * cclocks used in the CSR.
14095 * Limit the max if larger than the field holds. If timeout is
14096 * non-zero, then the calculated field will be at least 1.
14097 *
14098 * Must be after icode is set up - the cclock rate depends
14099 * on knowing the hardware being used.
14100 */
14101 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14102 if (dd->rcv_intr_timeout_csr >
14103 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14104 dd->rcv_intr_timeout_csr =
14105 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14106 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14107 dd->rcv_intr_timeout_csr = 1;
14108
7c03ed85
EH		 14109 /* needs to be done before we look for the peer device */
14110 read_guid(dd);
14111
14112 /* should this device init the ASIC block? */
14113 asic_should_init(dd);
14114
77241056
MM		 14115 /* obtain chip sizes, reset chip CSRs */
14116 init_chip(dd);
14117
14118 /* read in the PCIe link speed information */
14119 ret = pcie_speeds(dd);
14120 if (ret)
14121 goto bail_cleanup;
14122
c3838b39
EH		 14123 /* Needs to be called before hfi1_firmware_init */
14124 get_platform_config(dd);
14125
77241056
MM		 14126 /* read in firmware */
14127 ret = hfi1_firmware_init(dd);
14128 if (ret)
14129 goto bail_cleanup;
14130
14131 /*
14132 * In general, the PCIe Gen3 transition must occur after the
14133 * chip has been idled (so it won't initiate any PCIe transactions
14134 * e.g. an interrupt) and before the driver changes any registers
14135 * (the transition will reset the registers).
14136 *
14137 * In particular, place this call after:
14138 * - init_chip() - the chip will not initiate any PCIe transactions
14139 * - pcie_speeds() - reads the current link speed
14140 * - hfi1_firmware_init() - the needed firmware is ready to be
14141 * downloaded
14142 */
14143 ret = do_pcie_gen3_transition(dd);
14144 if (ret)
14145 goto bail_cleanup;
14146
14147 /* start setting dd values and adjusting CSRs */
14148 init_early_variables(dd);
14149
14150 parse_platform_config(dd);
14151
5d9157aa
DL		 14152 ret = obtain_boardname(dd);
14153 if (ret)
77241056 14154 goto bail_cleanup;
77241056
MM		 14155
14156 snprintf(dd->boardversion, BOARD_VERS_MAX,
5d9157aa 14157 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
77241056 14158 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
77241056
MM		 14159 (u32)dd->majrev,
14160 (u32)dd->minrev,
14161 (dd->revision >> CCE_REVISION_SW_SHIFT)
14162 & CCE_REVISION_SW_MASK);
14163
14164 ret = set_up_context_variables(dd);
14165 if (ret)
14166 goto bail_cleanup;
14167
14168 /* set initial RXE CSRs */
14169 init_rxe(dd);
14170 /* set initial TXE CSRs */
14171 init_txe(dd);
14172 /* set initial non-RXE, non-TXE CSRs */
14173 init_other(dd);
14174 /* set up KDETH QP prefix in both RX and TX CSRs */
14175 init_kdeth_qp(dd);
14176
957558c9
MH		 14177 ret = hfi1_dev_affinity_init(dd);
14178 if (ret)
14179 goto bail_cleanup;
14180
77241056
MM		 14181 /* send contexts must be set up before receive contexts */
14182 ret = init_send_contexts(dd);
14183 if (ret)
14184 goto bail_cleanup;
14185
14186 ret = hfi1_create_ctxts(dd);
14187 if (ret)
14188 goto bail_cleanup;
14189
14190 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
14191 /*
14192 * rcd[0] is guaranteed to be valid by this point. Also, all
14193 * context are using the same value, as per the module parameter.
14194 */
14195 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
14196
14197 ret = init_pervl_scs(dd);
14198 if (ret)
14199 goto bail_cleanup;
14200
14201 /* sdma init */
14202 for (i = 0; i < dd->num_pports; ++i) {
14203 ret = sdma_init(dd, i);
14204 if (ret)
14205 goto bail_cleanup;
14206 }
14207
14208 /* use contexts created by hfi1_create_ctxts */
14209 ret = set_up_interrupts(dd);
14210 if (ret)
14211 goto bail_cleanup;
14212
14213 /* set up LCB access - must be after set_up_interrupts() */
14214 init_lcb_access(dd);
14215
14216 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14217 dd->base_guid & 0xFFFFFF);
14218
14219 dd->oui1 = dd->base_guid >> 56 & 0xFF;
14220 dd->oui2 = dd->base_guid >> 48 & 0xFF;
14221 dd->oui3 = dd->base_guid >> 40 & 0xFF;
14222
14223 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14224 if (ret)
14225 goto bail_clear_intr;
14226 check_fabric_firmware_versions(dd);
14227
14228 thermal_init(dd);
14229
14230 ret = init_cntrs(dd);
14231 if (ret)
14232 goto bail_clear_intr;
14233
14234 ret = init_rcverr(dd);
14235 if (ret)
14236 goto bail_free_cntrs;
14237
14238 ret = eprom_init(dd);
14239 if (ret)
14240 goto bail_free_rcverr;
14241
14242 goto bail;
14243
14244bail_free_rcverr:
14245 free_rcverr(dd);
14246bail_free_cntrs:
14247 free_cntrs(dd);
14248bail_clear_intr:
14249 clean_up_interrupts(dd);
14250bail_cleanup:
14251 hfi1_pcie_ddcleanup(dd);
14252bail_free:
14253 hfi1_free_devdata(dd);
14254 dd = ERR_PTR(ret);
14255bail:
14256 return dd;
14257}
14258
14259static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
14260 u32 dw_len)
14261{
14262 u32 delta_cycles;
14263 u32 current_egress_rate = ppd->current_egress_rate;
14264 /* rates here are in units of 10^6 bits/sec */
14265
14266 if (desired_egress_rate == -1)
14267 return 0; /* shouldn't happen */
14268
14269 if (desired_egress_rate >= current_egress_rate)
14270 return 0; /* we can't help go faster, only slower */
14271
14272 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
14273 egress_cycles(dw_len * 4, current_egress_rate);
14274
14275 return (u16)delta_cycles;
14276}
14277
77241056
MM		 14278/**
14279 * create_pbc - build a pbc for transmission
14280 * @flags: special case flags or-ed in built pbc
14281 * @srate: static rate
14282 * @vl: vl
14283 * @dwlen: dword length (header words + data words + pbc words)
14284 *
14285 * Create a PBC with the given flags, rate, VL, and length.
14286 *
14287 * NOTE: The PBC created will not insert any HCRC - all callers but one are
14288 * for verbs, which does not use this PSM feature. The lone other caller
14289 * is for the diagnostic interface which calls this if the user does not
14290 * supply their own PBC.
14291 */
14292u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
14293 u32 dw_len)
14294{
14295 u64 pbc, delay = 0;
14296
14297 if (unlikely(srate_mbs))
14298 delay = delay_cycles(ppd, srate_mbs, dw_len);
14299
14300 pbc = flags
14301 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
14302 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
14303 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
14304 | (dw_len & PBC_LENGTH_DWS_MASK)
14305 << PBC_LENGTH_DWS_SHIFT;
14306
14307 return pbc;
14308}
14309
14310#define SBUS_THERMAL 0x4f
14311#define SBUS_THERM_MONITOR_MODE 0x1
14312
14313#define THERM_FAILURE(dev, ret, reason) \
14314 dd_dev_err((dd), \
14315 "Thermal sensor initialization failed: %s (%d)\n", \
14316 (reason), (ret))
14317
14318/*
14319 * Initialize the Avago Thermal sensor.
14320 *
14321 * After initialization, enable polling of thermal sensor through
14322 * SBus interface. In order for this to work, the SBus Master
14323 * firmware has to be loaded due to the fact that the HW polling
14324 * logic uses SBus interrupts, which are not supported with
14325 * default firmware. Otherwise, no data will be returned through
14326 * the ASIC_STS_THERM CSR.
14327 */
14328static int thermal_init(struct hfi1_devdata *dd)
14329{
14330 int ret = 0;
14331
14332 if (dd->icode != ICODE_RTL_SILICON ||
14333 !(dd->flags & HFI1_DO_INIT_ASIC))
14334 return ret;
14335
14336 acquire_hw_mutex(dd);
14337 dd_dev_info(dd, "Initializing thermal sensor\n");
4ef98989
JAQ		 14338 /* Disable polling of thermal readings */
14339 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
14340 msleep(100);
77241056
MM		 14341 /* Thermal Sensor Initialization */
14342 /* Step 1: Reset the Thermal SBus Receiver */
14343 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14344 RESET_SBUS_RECEIVER, 0);
14345 if (ret) {
14346 THERM_FAILURE(dd, ret, "Bus Reset");
14347 goto done;
14348 }
14349 /* Step 2: Set Reset bit in Thermal block */
14350 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14351 WRITE_SBUS_RECEIVER, 0x1);
14352 if (ret) {
14353 THERM_FAILURE(dd, ret, "Therm Block Reset");
14354 goto done;
14355 }
14356 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
14357 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
14358 WRITE_SBUS_RECEIVER, 0x32);
14359 if (ret) {
14360 THERM_FAILURE(dd, ret, "Write Clock Div");
14361 goto done;
14362 }
14363 /* Step 4: Select temperature mode */
14364 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
14365 WRITE_SBUS_RECEIVER,
14366 SBUS_THERM_MONITOR_MODE);
14367 if (ret) {
14368 THERM_FAILURE(dd, ret, "Write Mode Sel");
14369 goto done;
14370 }
14371 /* Step 5: De-assert block reset and start conversion */
14372 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14373 WRITE_SBUS_RECEIVER, 0x2);
14374 if (ret) {
14375 THERM_FAILURE(dd, ret, "Write Reset Deassert");
14376 goto done;
14377 }
14378 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
14379 msleep(22);
14380
14381 /* Enable polling of thermal readings */
14382 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
14383done:
14384 release_hw_mutex(dd);
14385 return ret;
14386}
14387
14388static void handle_temp_err(struct hfi1_devdata *dd)
14389{
14390 struct hfi1_pportdata *ppd = &dd->pport[0];
14391 /*
14392 * Thermal Critical Interrupt
14393 * Put the device into forced freeze mode, take link down to
14394 * offline, and put DC into reset.
14395 */
14396 dd_dev_emerg(dd,
14397 "Critical temperature reached! Forcing device into freeze mode!\n");
14398 dd->flags |= HFI1_FORCED_FREEZE;
8638b77f 14399 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
77241056
MM		 14400 /*
14401 * Shut DC down as much and as quickly as possible.
14402 *
14403 * Step 1: Take the link down to OFFLINE. This will cause the
14404 * 8051 to put the Serdes in reset. However, we don't want to
14405 * go through the entire link state machine since we want to
14406 * shutdown ASAP. Furthermore, this is not a graceful shutdown
14407 * but rather an attempt to save the chip.
14408 * Code below is almost the same as quiet_serdes() but avoids
14409 * all the extra work and the sleeps.
14410 */
14411 ppd->driver_link_ready = 0;
14412 ppd->link_enabled = 0;
14413 set_physical_link_state(dd, PLS_OFFLINE |
14414 (OPA_LINKDOWN_REASON_SMA_DISABLED << 8));
14415 /*
14416 * Step 2: Shutdown LCB and 8051
14417 * After shutdown, do not restore DC_CFG_RESET value.
14418 */
14419 dc_shutdown(dd);
14420}
Linux kernel - Deliverables
This page took 0.781111 seconds and 5 git commands to generate.