projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
staging/rdma/hfi1: rework is_a0() and is_bx()
[deliverable/linux.git] / drivers / staging / rdma / hfi1 / chip.c
CommitLineData
77241056
MM		 1/*
2 *
3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
5 *
6 * GPL LICENSE SUMMARY
7 *
8 * Copyright(c) 2015 Intel Corporation.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * BSD LICENSE
20 *
21 * Copyright(c) 2015 Intel Corporation.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 *
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
32 * distribution.
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 */
50
51/*
52 * This file contains all of the code that is specific to the HFI chip
53 */
54
55#include <linux/pci.h>
56#include <linux/delay.h>
57#include <linux/interrupt.h>
58#include <linux/module.h>
59
60#include "hfi.h"
61#include "trace.h"
62#include "mad.h"
63#include "pio.h"
64#include "sdma.h"
65#include "eprom.h"
66
67#define NUM_IB_PORTS 1
68
69uint kdeth_qp;
70module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
71MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
72
73uint num_vls = HFI1_MAX_VLS_SUPPORTED;
74module_param(num_vls, uint, S_IRUGO);
75MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
76
77/*
78 * Default time to aggregate two 10K packets from the idle state
79 * (timer not running). The timer starts at the end of the first packet,
80 * so only the time for one 10K packet and header plus a bit extra is needed.
81 * 10 * 1024 + 64 header byte = 10304 byte
82 * 10304 byte / 12.5 GB/s = 824.32ns
83 */
84uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
85module_param(rcv_intr_timeout, uint, S_IRUGO);
86MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
87
88uint rcv_intr_count = 16; /* same as qib */
89module_param(rcv_intr_count, uint, S_IRUGO);
90MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
91
92ushort link_crc_mask = SUPPORTED_CRCS;
93module_param(link_crc_mask, ushort, S_IRUGO);
94MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
95
96uint loopback;
97module_param_named(loopback, loopback, uint, S_IRUGO);
98MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
99
100/* Other driver tunables */
101uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
102static ushort crc_14b_sideband = 1;
103static uint use_flr = 1;
104uint quick_linkup; /* skip LNI */
105
106struct flag_table {
107 u64 flag; /* the flag */
108 char *str; /* description string */
109 u16 extra; /* extra information */
110 u16 unused0;
111 u32 unused1;
112};
113
114/* str must be a string constant */
115#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
116#define FLAG_ENTRY0(str, flag) {flag, str, 0}
117
118/* Send Error Consequences */
119#define SEC_WRITE_DROPPED 0x1
120#define SEC_PACKET_DROPPED 0x2
121#define SEC_SC_HALTED 0x4 /* per-context only */
122#define SEC_SPC_FREEZE 0x8 /* per-HFI only */
123
77241056 124#define MIN_KERNEL_KCTXTS 2
82c2611d 125#define FIRST_KERNEL_KCTXT 1
77241056
MM		 126#define NUM_MAP_REGS 32
127
128/* Bit offset into the GUID which carries HFI id information */
129#define GUID_HFI_INDEX_SHIFT 39
130
131/* extract the emulation revision */
132#define emulator_rev(dd) ((dd)->irev >> 8)
133/* parallel and serial emulation versions are 3 and 4 respectively */
134#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
135#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
136
137/* RSM fields */
138
139/* packet type */
140#define IB_PACKET_TYPE 2ull
141#define QW_SHIFT 6ull
142/* QPN[7..1] */
143#define QPN_WIDTH 7ull
144
145/* LRH.BTH: QW 0, OFFSET 48 - for match */
146#define LRH_BTH_QW 0ull
147#define LRH_BTH_BIT_OFFSET 48ull
148#define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
149#define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
150#define LRH_BTH_SELECT
151#define LRH_BTH_MASK 3ull
152#define LRH_BTH_VALUE 2ull
153
154/* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
155#define LRH_SC_QW 0ull
156#define LRH_SC_BIT_OFFSET 56ull
157#define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
158#define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
159#define LRH_SC_MASK 128ull
160#define LRH_SC_VALUE 0ull
161
162/* SC[n..0] QW 0, OFFSET 60 - for select */
163#define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
164
165/* QPN[m+n:1] QW 1, OFFSET 1 */
166#define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
167
168/* defines to build power on SC2VL table */
169#define SC2VL_VAL( \
170 num, \
171 sc0, sc0val, \
172 sc1, sc1val, \
173 sc2, sc2val, \
174 sc3, sc3val, \
175 sc4, sc4val, \
176 sc5, sc5val, \
177 sc6, sc6val, \
178 sc7, sc7val) \
179( \
180 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
181 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
182 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
183 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
184 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
185 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
186 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
187 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
188)
189
190#define DC_SC_VL_VAL( \
191 range, \
192 e0, e0val, \
193 e1, e1val, \
194 e2, e2val, \
195 e3, e3val, \
196 e4, e4val, \
197 e5, e5val, \
198 e6, e6val, \
199 e7, e7val, \
200 e8, e8val, \
201 e9, e9val, \
202 e10, e10val, \
203 e11, e11val, \
204 e12, e12val, \
205 e13, e13val, \
206 e14, e14val, \
207 e15, e15val) \
208( \
209 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
210 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
211 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
212 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
213 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
214 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
215 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
216 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
217 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
218 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
219 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
220 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
221 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
222 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
223 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
224 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
225)
226
227/* all CceStatus sub-block freeze bits */
228#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
229 | CCE_STATUS_RXE_FROZE_SMASK \
230 | CCE_STATUS_TXE_FROZE_SMASK \
231 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
232/* all CceStatus sub-block TXE pause bits */
233#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
234 | CCE_STATUS_TXE_PAUSED_SMASK \
235 | CCE_STATUS_SDMA_PAUSED_SMASK)
236/* all CceStatus sub-block RXE pause bits */
237#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
238
239/*
240 * CCE Error flags.
241 */
242static struct flag_table cce_err_status_flags[] = {
243/* 0*/ FLAG_ENTRY0("CceCsrParityErr",
244 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
245/* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
246 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
247/* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
248 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
249/* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
250 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
251/* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
252 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
253/* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
254 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
255/* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
256 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
257/* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
258 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
259/* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
260 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
261/* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
262 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
263/*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
264 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
265/*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
266 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
267/*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
268 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
269/*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
270 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
271/*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
272 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
273/*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
274 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
275/*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
276 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
277/*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
278 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
279/*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
280 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
281/*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
282 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
283/*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
284 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
285/*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
286 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
287/*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
288 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
289/*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
290 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
291/*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
292 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
293/*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
294 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
295/*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
296 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
297/*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
298 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
299/*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
300 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
301/*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
302 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
303/*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
304 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
305/*31*/ FLAG_ENTRY0("LATriggered",
306 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
307/*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
308 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
309/*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
310 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
311/*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
312 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
313/*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
314 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
315/*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
316 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
317/*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
318 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
319/*38*/ FLAG_ENTRY0("CceIntMapCorErr",
320 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
321/*39*/ FLAG_ENTRY0("CceIntMapUncErr",
322 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
323/*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
324 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
325/*41-63 reserved*/
326};
327
328/*
329 * Misc Error flags
330 */
331#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
332static struct flag_table misc_err_status_flags[] = {
333/* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
334/* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
335/* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
336/* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
337/* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
338/* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
339/* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
340/* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
341/* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
342/* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
343/*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
344/*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
345/*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
346};
347
348/*
349 * TXE PIO Error flags and consequences
350 */
351static struct flag_table pio_err_status_flags[] = {
352/* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
353 SEC_WRITE_DROPPED,
354 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
355/* 1*/ FLAG_ENTRY("PioWriteAddrParity",
356 SEC_SPC_FREEZE,
357 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
358/* 2*/ FLAG_ENTRY("PioCsrParity",
359 SEC_SPC_FREEZE,
360 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
361/* 3*/ FLAG_ENTRY("PioSbMemFifo0",
362 SEC_SPC_FREEZE,
363 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
364/* 4*/ FLAG_ENTRY("PioSbMemFifo1",
365 SEC_SPC_FREEZE,
366 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
367/* 5*/ FLAG_ENTRY("PioPccFifoParity",
368 SEC_SPC_FREEZE,
369 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
370/* 6*/ FLAG_ENTRY("PioPecFifoParity",
371 SEC_SPC_FREEZE,
372 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
373/* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
374 SEC_SPC_FREEZE,
375 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
376/* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
377 SEC_SPC_FREEZE,
378 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
379/* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
380 SEC_SPC_FREEZE,
381 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
382/*10*/ FLAG_ENTRY("PioSmPktResetParity",
383 SEC_SPC_FREEZE,
384 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
385/*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
386 SEC_SPC_FREEZE,
387 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
388/*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
389 SEC_SPC_FREEZE,
390 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
391/*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
392 0,
393 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
394/*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
395 0,
396 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
397/*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
398 SEC_SPC_FREEZE,
399 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
400/*16*/ FLAG_ENTRY("PioPpmcPblFifo",
401 SEC_SPC_FREEZE,
402 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
403/*17*/ FLAG_ENTRY("PioInitSmIn",
404 0,
405 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
406/*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
407 SEC_SPC_FREEZE,
408 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
409/*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
410 SEC_SPC_FREEZE,
411 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
412/*20*/ FLAG_ENTRY("PioHostAddrMemCor",
413 0,
414 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
415/*21*/ FLAG_ENTRY("PioWriteDataParity",
416 SEC_SPC_FREEZE,
417 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
418/*22*/ FLAG_ENTRY("PioStateMachine",
419 SEC_SPC_FREEZE,
420 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
421/*23*/ FLAG_ENTRY("PioWriteQwValidParity",
422 SEC_WRITE_DROPPED|SEC_SPC_FREEZE,
423 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
424/*24*/ FLAG_ENTRY("PioBlockQwCountParity",
425 SEC_WRITE_DROPPED|SEC_SPC_FREEZE,
426 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
427/*25*/ FLAG_ENTRY("PioVlfVlLenParity",
428 SEC_SPC_FREEZE,
429 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
430/*26*/ FLAG_ENTRY("PioVlfSopParity",
431 SEC_SPC_FREEZE,
432 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
433/*27*/ FLAG_ENTRY("PioVlFifoParity",
434 SEC_SPC_FREEZE,
435 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
436/*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
437 SEC_SPC_FREEZE,
438 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
439/*29*/ FLAG_ENTRY("PioPpmcSopLen",
440 SEC_SPC_FREEZE,
441 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
442/*30-31 reserved*/
443/*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
444 SEC_SPC_FREEZE,
445 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
446/*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
447 SEC_SPC_FREEZE,
448 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
449/*34*/ FLAG_ENTRY("PioPccSopHeadParity",
450 SEC_SPC_FREEZE,
451 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
452/*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
453 SEC_SPC_FREEZE,
454 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
455/*36-63 reserved*/
456};
457
458/* TXE PIO errors that cause an SPC freeze */
459#define ALL_PIO_FREEZE_ERR \
460 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
461 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
462 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
463 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
464 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
465 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
466 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
467 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
468 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
469 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
470 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
471 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
489
490/*
491 * TXE SDMA Error flags
492 */
493static struct flag_table sdma_err_status_flags[] = {
494/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
495 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
496/* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
497 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
498/* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
499 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
500/* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
501 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
502/*04-63 reserved*/
503};
504
505/* TXE SDMA errors that cause an SPC freeze */
506#define ALL_SDMA_FREEZE_ERR \
507 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
508 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
509 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
510
511/*
512 * TXE Egress Error flags
513 */
514#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
515static struct flag_table egress_err_status_flags[] = {
516/* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
517/* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
518/* 2 reserved */
519/* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
520 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
521/* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
522/* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
523/* 6 reserved */
524/* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
525 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
526/* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
527 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
528/* 9-10 reserved */
529/*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
530 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
531/*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
532/*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
533/*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
534/*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
535/*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
536 SEES(TX_SDMA0_DISALLOWED_PACKET)),
537/*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
538 SEES(TX_SDMA1_DISALLOWED_PACKET)),
539/*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
540 SEES(TX_SDMA2_DISALLOWED_PACKET)),
541/*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
542 SEES(TX_SDMA3_DISALLOWED_PACKET)),
543/*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
544 SEES(TX_SDMA4_DISALLOWED_PACKET)),
545/*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
546 SEES(TX_SDMA5_DISALLOWED_PACKET)),
547/*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
548 SEES(TX_SDMA6_DISALLOWED_PACKET)),
549/*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
550 SEES(TX_SDMA7_DISALLOWED_PACKET)),
551/*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
552 SEES(TX_SDMA8_DISALLOWED_PACKET)),
553/*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
554 SEES(TX_SDMA9_DISALLOWED_PACKET)),
555/*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
556 SEES(TX_SDMA10_DISALLOWED_PACKET)),
557/*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
558 SEES(TX_SDMA11_DISALLOWED_PACKET)),
559/*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
560 SEES(TX_SDMA12_DISALLOWED_PACKET)),
561/*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
562 SEES(TX_SDMA13_DISALLOWED_PACKET)),
563/*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
564 SEES(TX_SDMA14_DISALLOWED_PACKET)),
565/*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
566 SEES(TX_SDMA15_DISALLOWED_PACKET)),
567/*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
568 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
569/*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
570 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
571/*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
572 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
573/*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
574 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
575/*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
576 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
577/*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
578 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
579/*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
580 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
581/*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
582 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
583/*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
584 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
585/*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
586/*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
587/*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
588/*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
589/*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
590/*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
591/*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
592/*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
593/*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
594/*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
595/*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
596/*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
597/*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
598/*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
599/*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
600/*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
601/*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
602/*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
603/*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
604/*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
605/*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
606/*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
607 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
608/*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
609 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
610};
611
612/*
613 * TXE Egress Error Info flags
614 */
615#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
616static struct flag_table egress_err_info_flags[] = {
617/* 0*/ FLAG_ENTRY0("Reserved", 0ull),
618/* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
619/* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
620/* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
621/* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
622/* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
623/* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
624/* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
625/* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
626/* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
627/*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
628/*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
629/*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
630/*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
631/*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
632/*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
633/*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
634/*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
635/*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
636/*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
637/*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
638/*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
639};
640
641/* TXE Egress errors that cause an SPC freeze */
642#define ALL_TXE_EGRESS_FREEZE_ERR \
643 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
644 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
645 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
646 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
647 | SEES(TX_LAUNCH_CSR_PARITY) \
648 | SEES(TX_SBRD_CTL_CSR_PARITY) \
649 | SEES(TX_CONFIG_PARITY) \
650 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
651 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
652 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
653 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
654 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
655 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
656 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
657 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
658 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
659 | SEES(TX_CREDIT_RETURN_PARITY))
660
661/*
662 * TXE Send error flags
663 */
664#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
665static struct flag_table send_err_status_flags[] = {
666/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", SES(CSR_PARITY)),
667/* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
668/* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
669};
670
671/*
672 * TXE Send Context Error flags and consequences
673 */
674static struct flag_table sc_err_status_flags[] = {
675/* 0*/ FLAG_ENTRY("InconsistentSop",
676 SEC_PACKET_DROPPED | SEC_SC_HALTED,
677 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
678/* 1*/ FLAG_ENTRY("DisallowedPacket",
679 SEC_PACKET_DROPPED | SEC_SC_HALTED,
680 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
681/* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
682 SEC_WRITE_DROPPED | SEC_SC_HALTED,
683 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
684/* 3*/ FLAG_ENTRY("WriteOverflow",
685 SEC_WRITE_DROPPED | SEC_SC_HALTED,
686 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
687/* 4*/ FLAG_ENTRY("WriteOutOfBounds",
688 SEC_WRITE_DROPPED | SEC_SC_HALTED,
689 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
690/* 5-63 reserved*/
691};
692
693/*
694 * RXE Receive Error flags
695 */
696#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
697static struct flag_table rxe_err_status_flags[] = {
698/* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
699/* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
700/* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
701/* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
702/* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
703/* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
704/* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
705/* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
706/* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
707/* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
708/*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
709/*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
710/*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
711/*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
712/*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
713/*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
714/*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
715 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
716/*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
717/*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
718/*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
719 RXES(RBUF_BLOCK_LIST_READ_UNC)),
720/*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
721 RXES(RBUF_BLOCK_LIST_READ_COR)),
722/*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
723 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
724/*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
725 RXES(RBUF_CSR_QENT_CNT_PARITY)),
726/*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
727 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
728/*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
729 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
730/*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
731/*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
732/*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
733 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
734/*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
735/*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
736/*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
737/*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
738/*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
739/*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
740/*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
741/*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
742 RXES(RBUF_FL_INITDONE_PARITY)),
743/*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
744 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
745/*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
746/*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
747/*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
748/*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
749 RXES(LOOKUP_DES_PART1_UNC_COR)),
750/*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
751 RXES(LOOKUP_DES_PART2_PARITY)),
752/*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
753/*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
754/*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
755/*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
756/*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
757/*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
758/*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
759/*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
760/*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
761/*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
762/*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
763/*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
764/*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
765/*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
766/*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
767/*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
768/*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
769/*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
770/*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
771/*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
772/*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
773/*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
774};
775
776/* RXE errors that will trigger an SPC freeze */
777#define ALL_RXE_FREEZE_ERR \
778 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
779 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
780 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
781 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
782 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
783 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
784 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
785 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
786 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
787 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
788 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
789 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
790 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
791 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
792 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
793 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
794 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
795 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
822
823#define RXE_FREEZE_ABORT_MASK \
824 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
825 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
826 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
827
828/*
829 * DCC Error Flags
830 */
831#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
832static struct flag_table dcc_err_flags[] = {
833 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
834 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
835 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
836 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
837 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
838 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
839 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
840 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
841 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
842 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
843 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
844 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
845 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
846 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
847 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
848 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
849 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
850 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
851 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
852 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
853 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
854 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
855 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
856 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
857 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
858 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
859 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
860 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
861 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
862 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
863 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
864 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
865 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
866 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
867 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
868 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
869 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
870 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
871 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
872 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
873 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
874 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
875 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
876 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
877 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
878 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
879};
880
881/*
882 * LCB error flags
883 */
884#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
885static struct flag_table lcb_err_flags[] = {
886/* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
887/* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
888/* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
889/* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
890 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
891/* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
892/* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
893/* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
894/* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
895/* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
896/* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
897/*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
898/*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
899/*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
900/*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
901 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
902/*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
903/*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
904/*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
905/*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
906/*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
907/*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
908 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
909/*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
910/*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
911/*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
912/*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
913/*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
914/*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
915/*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
916 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
917/*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
918/*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
919 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
920/*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
921 LCBE(REDUNDANT_FLIT_PARITY_ERR))
922};
923
924/*
925 * DC8051 Error Flags
926 */
927#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
928static struct flag_table dc8051_err_flags[] = {
929 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
930 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
931 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
932 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
933 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
934 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
935 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
936 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
937 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
938 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
939 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
940};
941
942/*
943 * DC8051 Information Error flags
944 *
945 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
946 */
947static struct flag_table dc8051_info_err_flags[] = {
948 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
949 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
950 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
951 FLAG_ENTRY0("Serdes internal loopback failure",
952 FAILED_SERDES_INTERNAL_LOOPBACK),
953 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
954 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
955 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
956 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
957 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
958 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
959 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
960 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT)
961};
962
963/*
964 * DC8051 Information Host Information flags
965 *
966 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
967 */
968static struct flag_table dc8051_info_host_msg_flags[] = {
969 FLAG_ENTRY0("Host request done", 0x0001),
970 FLAG_ENTRY0("BC SMA message", 0x0002),
971 FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
972 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
973 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
974 FLAG_ENTRY0("External device config request", 0x0020),
975 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
976 FLAG_ENTRY0("LinkUp achieved", 0x0080),
977 FLAG_ENTRY0("Link going down", 0x0100),
978};
979
980
981static u32 encoded_size(u32 size);
982static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
983static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
984static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
985 u8 *continuous);
986static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
987 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
988static void read_vc_remote_link_width(struct hfi1_devdata *dd,
989 u8 *remote_tx_rate, u16 *link_widths);
990static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
991 u8 *flag_bits, u16 *link_widths);
992static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
993 u8 *device_rev);
994static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
995static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
996static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
997 u8 *tx_polarity_inversion,
998 u8 *rx_polarity_inversion, u8 *max_rate);
999static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1000 unsigned int context, u64 err_status);
1001static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1002static void handle_dcc_err(struct hfi1_devdata *dd,
1003 unsigned int context, u64 err_status);
1004static void handle_lcb_err(struct hfi1_devdata *dd,
1005 unsigned int context, u64 err_status);
1006static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1007static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1008static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1009static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1010static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1011static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1012static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1013static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1014static void set_partition_keys(struct hfi1_pportdata *);
1015static const char *link_state_name(u32 state);
1016static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1017 u32 state);
1018static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1019 u64 *out_data);
1020static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1021static int thermal_init(struct hfi1_devdata *dd);
1022
1023static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1024 int msecs);
1025static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1026static void handle_temp_err(struct hfi1_devdata *);
1027static void dc_shutdown(struct hfi1_devdata *);
1028static void dc_start(struct hfi1_devdata *);
1029
1030/*
1031 * Error interrupt table entry. This is used as input to the interrupt
1032 * "clear down" routine used for all second tier error interrupt register.
1033 * Second tier interrupt registers have a single bit representing them
1034 * in the top-level CceIntStatus.
1035 */
1036struct err_reg_info {
1037 u32 status; /* status CSR offset */
1038 u32 clear; /* clear CSR offset */
1039 u32 mask; /* mask CSR offset */
1040 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1041 const char *desc;
1042};
1043
1044#define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1045#define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1046#define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1047
1048/*
1049 * Helpers for building HFI and DC error interrupt table entries. Different
1050 * helpers are needed because of inconsistent register names.
1051 */
1052#define EE(reg, handler, desc) \
1053 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1054 handler, desc }
1055#define DC_EE1(reg, handler, desc) \
1056 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1057#define DC_EE2(reg, handler, desc) \
1058 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1059
1060/*
1061 * Table of the "misc" grouping of error interrupts. Each entry refers to
1062 * another register containing more information.
1063 */
1064static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1065/* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1066/* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1067/* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1068/* 3*/ { 0, 0, 0, NULL }, /* reserved */
1069/* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1070/* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1071/* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1072/* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1073 /* the rest are reserved */
1074};
1075
1076/*
1077 * Index into the Various section of the interrupt sources
1078 * corresponding to the Critical Temperature interrupt.
1079 */
1080#define TCRIT_INT_SOURCE 4
1081
1082/*
1083 * SDMA error interrupt entry - refers to another register containing more
1084 * information.
1085 */
1086static const struct err_reg_info sdma_eng_err =
1087 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1088
1089static const struct err_reg_info various_err[NUM_VARIOUS] = {
1090/* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1091/* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1092/* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1093/* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1094/* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1095 /* rest are reserved */
1096};
1097
1098/*
1099 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1100 * register can not be derived from the MTU value because 10K is not
1101 * a power of 2. Therefore, we need a constant. Everything else can
1102 * be calculated.
1103 */
1104#define DCC_CFG_PORT_MTU_CAP_10240 7
1105
1106/*
1107 * Table of the DC grouping of error interrupts. Each entry refers to
1108 * another register containing more information.
1109 */
1110static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1111/* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1112/* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1113/* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1114/* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1115 /* the rest are reserved */
1116};
1117
1118struct cntr_entry {
1119 /*
1120 * counter name
1121 */
1122 char *name;
1123
1124 /*
1125 * csr to read for name (if applicable)
1126 */
1127 u64 csr;
1128
1129 /*
1130 * offset into dd or ppd to store the counter's value
1131 */
1132 int offset;
1133
1134 /*
1135 * flags
1136 */
1137 u8 flags;
1138
1139 /*
1140 * accessor for stat element, context either dd or ppd
1141 */
1142 u64 (*rw_cntr)(const struct cntr_entry *,
1143 void *context,
1144 int vl,
1145 int mode,
1146 u64 data);
1147};
1148
1149#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1150#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1151
1152#define CNTR_ELEM(name, csr, offset, flags, accessor) \
1153{ \
1154 name, \
1155 csr, \
1156 offset, \
1157 flags, \
1158 accessor \
1159}
1160
1161/* 32bit RXE */
1162#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1163CNTR_ELEM(#name, \
1164 (counter * 8 + RCV_COUNTER_ARRAY32), \
1165 0, flags | CNTR_32BIT, \
1166 port_access_u32_csr)
1167
1168#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1169CNTR_ELEM(#name, \
1170 (counter * 8 + RCV_COUNTER_ARRAY32), \
1171 0, flags | CNTR_32BIT, \
1172 dev_access_u32_csr)
1173
1174/* 64bit RXE */
1175#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1176CNTR_ELEM(#name, \
1177 (counter * 8 + RCV_COUNTER_ARRAY64), \
1178 0, flags, \
1179 port_access_u64_csr)
1180
1181#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1182CNTR_ELEM(#name, \
1183 (counter * 8 + RCV_COUNTER_ARRAY64), \
1184 0, flags, \
1185 dev_access_u64_csr)
1186
1187#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1188#define OVR_ELM(ctx) \
1189CNTR_ELEM("RcvHdrOvr" #ctx, \
1190 (RCV_HDR_OVFL_CNT + ctx*0x100), \
1191 0, CNTR_NORMAL, port_access_u64_csr)
1192
1193/* 32bit TXE */
1194#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1195CNTR_ELEM(#name, \
1196 (counter * 8 + SEND_COUNTER_ARRAY32), \
1197 0, flags | CNTR_32BIT, \
1198 port_access_u32_csr)
1199
1200/* 64bit TXE */
1201#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1202CNTR_ELEM(#name, \
1203 (counter * 8 + SEND_COUNTER_ARRAY64), \
1204 0, flags, \
1205 port_access_u64_csr)
1206
1207# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1208CNTR_ELEM(#name,\
1209 counter * 8 + SEND_COUNTER_ARRAY64, \
1210 0, \
1211 flags, \
1212 dev_access_u64_csr)
1213
1214/* CCE */
1215#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1216CNTR_ELEM(#name, \
1217 (counter * 8 + CCE_COUNTER_ARRAY32), \
1218 0, flags | CNTR_32BIT, \
1219 dev_access_u32_csr)
1220
1221#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1222CNTR_ELEM(#name, \
1223 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1224 0, flags | CNTR_32BIT, \
1225 dev_access_u32_csr)
1226
1227/* DC */
1228#define DC_PERF_CNTR(name, counter, flags) \
1229CNTR_ELEM(#name, \
1230 counter, \
1231 0, \
1232 flags, \
1233 dev_access_u64_csr)
1234
1235#define DC_PERF_CNTR_LCB(name, counter, flags) \
1236CNTR_ELEM(#name, \
1237 counter, \
1238 0, \
1239 flags, \
1240 dc_access_lcb_cntr)
1241
1242/* ibp counters */
1243#define SW_IBP_CNTR(name, cntr) \
1244CNTR_ELEM(#name, \
1245 0, \
1246 0, \
1247 CNTR_SYNTH, \
1248 access_ibp_##cntr)
1249
1250u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1251{
1252 u64 val;
1253
1254 if (dd->flags & HFI1_PRESENT) {
1255 val = readq((void __iomem *)dd->kregbase + offset);
1256 return val;
1257 }
1258 return -1;
1259}
1260
1261void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1262{
1263 if (dd->flags & HFI1_PRESENT)
1264 writeq(value, (void __iomem *)dd->kregbase + offset);
1265}
1266
1267void __iomem *get_csr_addr(
1268 struct hfi1_devdata *dd,
1269 u32 offset)
1270{
1271 return (void __iomem *)dd->kregbase + offset;
1272}
1273
1274static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1275 int mode, u64 value)
1276{
1277 u64 ret;
1278
1279
1280 if (mode == CNTR_MODE_R) {
1281 ret = read_csr(dd, csr);
1282 } else if (mode == CNTR_MODE_W) {
1283 write_csr(dd, csr, value);
1284 ret = value;
1285 } else {
1286 dd_dev_err(dd, "Invalid cntr register access mode");
1287 return 0;
1288 }
1289
1290 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1291 return ret;
1292}
1293
1294/* Dev Access */
1295static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1296 void *context, int vl, int mode, u64 data)
1297{
a787bde8 1298 struct hfi1_devdata *dd = context;
77241056
MM		 1299
1300 if (vl != CNTR_INVALID_VL)
1301 return 0;
1302 return read_write_csr(dd, entry->csr, mode, data);
1303}
1304
1305static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1306 int vl, int mode, u64 data)
1307{
a787bde8 1308 struct hfi1_devdata *dd = context;
77241056
MM		 1309
1310 u64 val = 0;
1311 u64 csr = entry->csr;
1312
1313 if (entry->flags & CNTR_VL) {
1314 if (vl == CNTR_INVALID_VL)
1315 return 0;
1316 csr += 8 * vl;
1317 } else {
1318 if (vl != CNTR_INVALID_VL)
1319 return 0;
1320 }
1321
1322 val = read_write_csr(dd, csr, mode, data);
1323 return val;
1324}
1325
1326static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1327 int vl, int mode, u64 data)
1328{
a787bde8 1329 struct hfi1_devdata *dd = context;
77241056
MM		 1330 u32 csr = entry->csr;
1331 int ret = 0;
1332
1333 if (vl != CNTR_INVALID_VL)
1334 return 0;
1335 if (mode == CNTR_MODE_R)
1336 ret = read_lcb_csr(dd, csr, &data);
1337 else if (mode == CNTR_MODE_W)
1338 ret = write_lcb_csr(dd, csr, data);
1339
1340 if (ret) {
1341 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1342 return 0;
1343 }
1344
1345 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1346 return data;
1347}
1348
1349/* Port Access */
1350static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1351 int vl, int mode, u64 data)
1352{
a787bde8 1353 struct hfi1_pportdata *ppd = context;
77241056
MM		 1354
1355 if (vl != CNTR_INVALID_VL)
1356 return 0;
1357 return read_write_csr(ppd->dd, entry->csr, mode, data);
1358}
1359
1360static u64 port_access_u64_csr(const struct cntr_entry *entry,
1361 void *context, int vl, int mode, u64 data)
1362{
a787bde8 1363 struct hfi1_pportdata *ppd = context;
77241056
MM		 1364 u64 val;
1365 u64 csr = entry->csr;
1366
1367 if (entry->flags & CNTR_VL) {
1368 if (vl == CNTR_INVALID_VL)
1369 return 0;
1370 csr += 8 * vl;
1371 } else {
1372 if (vl != CNTR_INVALID_VL)
1373 return 0;
1374 }
1375 val = read_write_csr(ppd->dd, csr, mode, data);
1376 return val;
1377}
1378
1379/* Software defined */
1380static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1381 u64 data)
1382{
1383 u64 ret;
1384
1385 if (mode == CNTR_MODE_R) {
1386 ret = *cntr;
1387 } else if (mode == CNTR_MODE_W) {
1388 *cntr = data;
1389 ret = data;
1390 } else {
1391 dd_dev_err(dd, "Invalid cntr sw access mode");
1392 return 0;
1393 }
1394
1395 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1396
1397 return ret;
1398}
1399
1400static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1401 int vl, int mode, u64 data)
1402{
a787bde8 1403 struct hfi1_pportdata *ppd = context;
77241056
MM		 1404
1405 if (vl != CNTR_INVALID_VL)
1406 return 0;
1407 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1408}
1409
1410static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1411 int vl, int mode, u64 data)
1412{
a787bde8 1413 struct hfi1_pportdata *ppd = context;
77241056
MM		 1414
1415 if (vl != CNTR_INVALID_VL)
1416 return 0;
1417 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1418}
1419
1420static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1421 void *context, int vl, int mode, u64 data)
1422{
a787bde8 1423 struct hfi1_pportdata *ppd = context;
77241056
MM		 1424
1425 if (vl != CNTR_INVALID_VL)
1426 return 0;
1427
1428 return read_write_sw(ppd->dd, &ppd->port_xmit_discards, mode, data);
1429}
1430
1431static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1432 void *context, int vl, int mode, u64 data)
1433{
a787bde8 1434 struct hfi1_pportdata *ppd = context;
77241056
MM		 1435
1436 if (vl != CNTR_INVALID_VL)
1437 return 0;
1438
1439 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1440 mode, data);
1441}
1442
1443static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1444 void *context, int vl, int mode, u64 data)
1445{
a787bde8 1446 struct hfi1_pportdata *ppd = context;
77241056
MM		 1447
1448 if (vl != CNTR_INVALID_VL)
1449 return 0;
1450
1451 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1452 mode, data);
1453}
1454
1455u64 get_all_cpu_total(u64 __percpu *cntr)
1456{
1457 int cpu;
1458 u64 counter = 0;
1459
1460 for_each_possible_cpu(cpu)
1461 counter += *per_cpu_ptr(cntr, cpu);
1462 return counter;
1463}
1464
1465static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1466 u64 __percpu *cntr,
1467 int vl, int mode, u64 data)
1468{
1469
1470 u64 ret = 0;
1471
1472 if (vl != CNTR_INVALID_VL)
1473 return 0;
1474
1475 if (mode == CNTR_MODE_R) {
1476 ret = get_all_cpu_total(cntr) - *z_val;
1477 } else if (mode == CNTR_MODE_W) {
1478 /* A write can only zero the counter */
1479 if (data == 0)
1480 *z_val = get_all_cpu_total(cntr);
1481 else
1482 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1483 } else {
1484 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1485 return 0;
1486 }
1487
1488 return ret;
1489}
1490
1491static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1492 void *context, int vl, int mode, u64 data)
1493{
a787bde8 1494 struct hfi1_devdata *dd = context;
77241056
MM		 1495
1496 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1497 mode, data);
1498}
1499
1500static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1501 void *context, int vl, int mode, u64 data)
1502{
a787bde8 1503 struct hfi1_devdata *dd = context;
77241056
MM		 1504
1505 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1506 mode, data);
1507}
1508
1509static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1510 void *context, int vl, int mode, u64 data)
1511{
a787bde8 1512 struct hfi1_devdata *dd = context;
77241056
MM		 1513
1514 return dd->verbs_dev.n_piowait;
1515}
1516
1517static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1518 void *context, int vl, int mode, u64 data)
1519{
a787bde8 1520 struct hfi1_devdata *dd = context;
77241056
MM		 1521
1522 return dd->verbs_dev.n_txwait;
1523}
1524
1525static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1526 void *context, int vl, int mode, u64 data)
1527{
a787bde8 1528 struct hfi1_devdata *dd = context;
77241056
MM		 1529
1530 return dd->verbs_dev.n_kmem_wait;
1531}
1532
b421922e
DL		 1533static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1534 void *context, int vl, int mode, u64 data)
1535{
1536 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1537
1538 return dd->verbs_dev.n_send_schedule;
1539}
1540
77241056
MM		 1541#define def_access_sw_cpu(cntr) \
1542static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
1543 void *context, int vl, int mode, u64 data) \
1544{ \
1545 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
1546 return read_write_cpu(ppd->dd, &ppd->ibport_data.z_ ##cntr, \
1547 ppd->ibport_data.cntr, vl, \
1548 mode, data); \
1549}
1550
1551def_access_sw_cpu(rc_acks);
1552def_access_sw_cpu(rc_qacks);
1553def_access_sw_cpu(rc_delayed_comp);
1554
1555#define def_access_ibp_counter(cntr) \
1556static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
1557 void *context, int vl, int mode, u64 data) \
1558{ \
1559 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
1560 \
1561 if (vl != CNTR_INVALID_VL) \
1562 return 0; \
1563 \
1564 return read_write_sw(ppd->dd, &ppd->ibport_data.n_ ##cntr, \
1565 mode, data); \
1566}
1567
1568def_access_ibp_counter(loop_pkts);
1569def_access_ibp_counter(rc_resends);
1570def_access_ibp_counter(rnr_naks);
1571def_access_ibp_counter(other_naks);
1572def_access_ibp_counter(rc_timeouts);
1573def_access_ibp_counter(pkt_drops);
1574def_access_ibp_counter(dmawait);
1575def_access_ibp_counter(rc_seqnak);
1576def_access_ibp_counter(rc_dupreq);
1577def_access_ibp_counter(rdma_seq);
1578def_access_ibp_counter(unaligned);
1579def_access_ibp_counter(seq_naks);
1580
1581static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
1582[C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
1583[C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
1584 CNTR_NORMAL),
1585[C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
1586 CNTR_NORMAL),
1587[C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
1588 RCV_TID_FLOW_GEN_MISMATCH_CNT,
1589 CNTR_NORMAL),
77241056
MM		 1590[C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
1591 CNTR_NORMAL),
1592[C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
1593 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
1594[C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
1595 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
1596[C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
1597 CNTR_NORMAL),
1598[C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
1599 CNTR_NORMAL),
1600[C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
1601 CNTR_NORMAL),
1602[C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
1603 CNTR_NORMAL),
1604[C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
1605 CNTR_NORMAL),
1606[C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
1607 CNTR_NORMAL),
1608[C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
1609 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
1610[C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
1611 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
1612[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
1613 CNTR_SYNTH),
1614[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
1615[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
1616 CNTR_SYNTH),
1617[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
1618 CNTR_SYNTH),
1619[C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
1620 CNTR_SYNTH),
1621[C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
1622 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
1623[C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
1624 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
1625 CNTR_SYNTH),
1626[C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
1627 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
1628[C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
1629 CNTR_SYNTH),
1630[C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
1631 CNTR_SYNTH),
1632[C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
1633 CNTR_SYNTH),
1634[C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
1635 CNTR_SYNTH),
1636[C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
1637 CNTR_SYNTH),
1638[C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
1639 CNTR_SYNTH),
1640[C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
1641 CNTR_SYNTH),
1642[C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
1643 CNTR_SYNTH | CNTR_VL),
1644[C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
1645 CNTR_SYNTH | CNTR_VL),
1646[C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
1647[C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
1648 CNTR_SYNTH | CNTR_VL),
1649[C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
1650[C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
1651 CNTR_SYNTH | CNTR_VL),
1652[C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
1653 CNTR_SYNTH),
1654[C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
1655 CNTR_SYNTH | CNTR_VL),
1656[C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
1657 CNTR_SYNTH),
1658[C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
1659 CNTR_SYNTH | CNTR_VL),
1660[C_DC_TOTAL_CRC] =
1661 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
1662 CNTR_SYNTH),
1663[C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
1664 CNTR_SYNTH),
1665[C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
1666 CNTR_SYNTH),
1667[C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
1668 CNTR_SYNTH),
1669[C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
1670 CNTR_SYNTH),
1671[C_DC_CRC_MULT_LN] =
1672 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
1673 CNTR_SYNTH),
1674[C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
1675 CNTR_SYNTH),
1676[C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
1677 CNTR_SYNTH),
1678[C_DC_SEQ_CRC_CNT] =
1679 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
1680 CNTR_SYNTH),
1681[C_DC_ESC0_ONLY_CNT] =
1682 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
1683 CNTR_SYNTH),
1684[C_DC_ESC0_PLUS1_CNT] =
1685 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
1686 CNTR_SYNTH),
1687[C_DC_ESC0_PLUS2_CNT] =
1688 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
1689 CNTR_SYNTH),
1690[C_DC_REINIT_FROM_PEER_CNT] =
1691 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
1692 CNTR_SYNTH),
1693[C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
1694 CNTR_SYNTH),
1695[C_DC_MISC_FLG_CNT] =
1696 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
1697 CNTR_SYNTH),
1698[C_DC_PRF_GOOD_LTP_CNT] =
1699 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
1700[C_DC_PRF_ACCEPTED_LTP_CNT] =
1701 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
1702 CNTR_SYNTH),
1703[C_DC_PRF_RX_FLIT_CNT] =
1704 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
1705[C_DC_PRF_TX_FLIT_CNT] =
1706 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
1707[C_DC_PRF_CLK_CNTR] =
1708 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
1709[C_DC_PG_DBG_FLIT_CRDTS_CNT] =
1710 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
1711[C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
1712 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
1713 CNTR_SYNTH),
1714[C_DC_PG_STS_TX_SBE_CNT] =
1715 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
1716[C_DC_PG_STS_TX_MBE_CNT] =
1717 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
1718 CNTR_SYNTH),
1719[C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
1720 access_sw_cpu_intr),
1721[C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
1722 access_sw_cpu_rcv_limit),
1723[C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
1724 access_sw_vtx_wait),
1725[C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
1726 access_sw_pio_wait),
1727[C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
1728 access_sw_kmem_wait),
b421922e
DL		 1729[C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
1730 access_sw_send_schedule),
77241056
MM		 1731};
1732
1733static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
1734[C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
1735 CNTR_NORMAL),
1736[C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
1737 CNTR_NORMAL),
1738[C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
1739 CNTR_NORMAL),
1740[C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
1741 CNTR_NORMAL),
1742[C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
1743 CNTR_NORMAL),
1744[C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
1745 CNTR_NORMAL),
1746[C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
1747 CNTR_NORMAL),
1748[C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
1749[C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
1750[C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
1751[C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
1752 CNTR_SYNTH | CNTR_VL),
1753[C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
1754 CNTR_SYNTH | CNTR_VL),
1755[C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
1756 CNTR_SYNTH | CNTR_VL),
1757[C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
1758[C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
1759[C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1760 access_sw_link_dn_cnt),
1761[C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1762 access_sw_link_up_cnt),
1763[C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1764 access_sw_xmit_discards),
1765[C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
1766 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
1767 access_sw_xmit_discards),
1768[C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
1769 access_xmit_constraint_errs),
1770[C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
1771 access_rcv_constraint_errs),
1772[C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
1773[C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
1774[C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
1775[C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
1776[C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
1777[C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
1778[C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
1779[C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
1780[C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
1781[C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
1782[C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
1783[C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
1784[C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
1785 access_sw_cpu_rc_acks),
1786[C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
1787 access_sw_cpu_rc_qacks),
1788[C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
1789 access_sw_cpu_rc_delayed_comp),
1790[OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
1791[OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
1792[OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
1793[OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
1794[OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
1795[OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
1796[OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
1797[OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
1798[OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
1799[OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
1800[OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
1801[OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
1802[OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
1803[OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
1804[OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
1805[OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
1806[OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
1807[OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
1808[OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
1809[OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
1810[OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
1811[OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
1812[OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
1813[OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
1814[OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
1815[OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
1816[OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
1817[OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
1818[OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
1819[OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
1820[OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
1821[OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
1822[OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
1823[OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
1824[OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
1825[OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
1826[OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
1827[OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
1828[OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
1829[OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
1830[OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
1831[OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
1832[OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
1833[OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
1834[OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
1835[OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
1836[OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
1837[OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
1838[OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
1839[OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
1840[OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
1841[OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
1842[OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
1843[OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
1844[OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
1845[OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
1846[OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
1847[OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
1848[OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
1849[OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
1850[OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
1851[OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
1852[OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
1853[OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
1854[OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
1855[OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
1856[OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
1857[OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
1858[OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
1859[OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
1860[OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
1861[OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
1862[OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
1863[OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
1864[OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
1865[OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
1866[OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
1867[OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
1868[OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
1869[OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
1870};
1871
1872/* ======================================================================== */
1873
77241056
MM		 1874/* return true if this is chip revision revision a */
1875int is_ax(struct hfi1_devdata *dd)
1876{
1877 u8 chip_rev_minor =
1878 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
1879 & CCE_REVISION_CHIP_REV_MINOR_MASK;
1880 return (chip_rev_minor & 0xf0) == 0;
1881}
1882
1883/* return true if this is chip revision revision b */
1884int is_bx(struct hfi1_devdata *dd)
1885{
1886 u8 chip_rev_minor =
1887 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
1888 & CCE_REVISION_CHIP_REV_MINOR_MASK;
995deafa 1889 return (chip_rev_minor & 0xF0) == 0x10;
77241056
MM		 1890}
1891
1892/*
1893 * Append string s to buffer buf. Arguments curp and len are the current
1894 * position and remaining length, respectively.
1895 *
1896 * return 0 on success, 1 on out of room
1897 */
1898static int append_str(char *buf, char **curp, int *lenp, const char *s)
1899{
1900 char *p = *curp;
1901 int len = *lenp;
1902 int result = 0; /* success */
1903 char c;
1904
1905 /* add a comma, if first in the buffer */
1906 if (p != buf) {
1907 if (len == 0) {
1908 result = 1; /* out of room */
1909 goto done;
1910 }
1911 *p++ = ',';
1912 len--;
1913 }
1914
1915 /* copy the string */
1916 while ((c = *s++) != 0) {
1917 if (len == 0) {
1918 result = 1; /* out of room */
1919 goto done;
1920 }
1921 *p++ = c;
1922 len--;
1923 }
1924
1925done:
1926 /* write return values */
1927 *curp = p;
1928 *lenp = len;
1929
1930 return result;
1931}
1932
1933/*
1934 * Using the given flag table, print a comma separated string into
1935 * the buffer. End in '*' if the buffer is too short.
1936 */
1937static char *flag_string(char *buf, int buf_len, u64 flags,
1938 struct flag_table *table, int table_size)
1939{
1940 char extra[32];
1941 char *p = buf;
1942 int len = buf_len;
1943 int no_room = 0;
1944 int i;
1945
1946 /* make sure there is at least 2 so we can form "*" */
1947 if (len < 2)
1948 return "";
1949
1950 len--; /* leave room for a nul */
1951 for (i = 0; i < table_size; i++) {
1952 if (flags & table[i].flag) {
1953 no_room = append_str(buf, &p, &len, table[i].str);
1954 if (no_room)
1955 break;
1956 flags &= ~table[i].flag;
1957 }
1958 }
1959
1960 /* any undocumented bits left? */
1961 if (!no_room && flags) {
1962 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
1963 no_room = append_str(buf, &p, &len, extra);
1964 }
1965
1966 /* add * if ran out of room */
1967 if (no_room) {
1968 /* may need to back up to add space for a '*' */
1969 if (len == 0)
1970 --p;
1971 *p++ = '*';
1972 }
1973
1974 /* add final nul - space already allocated above */
1975 *p = 0;
1976 return buf;
1977}
1978
1979/* first 8 CCE error interrupt source names */
1980static const char * const cce_misc_names[] = {
1981 "CceErrInt", /* 0 */
1982 "RxeErrInt", /* 1 */
1983 "MiscErrInt", /* 2 */
1984 "Reserved3", /* 3 */
1985 "PioErrInt", /* 4 */
1986 "SDmaErrInt", /* 5 */
1987 "EgressErrInt", /* 6 */
1988 "TxeErrInt" /* 7 */
1989};
1990
1991/*
1992 * Return the miscellaneous error interrupt name.
1993 */
1994static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
1995{
1996 if (source < ARRAY_SIZE(cce_misc_names))
1997 strncpy(buf, cce_misc_names[source], bsize);
1998 else
1999 snprintf(buf,
2000 bsize,
2001 "Reserved%u",
2002 source + IS_GENERAL_ERR_START);
2003
2004 return buf;
2005}
2006
2007/*
2008 * Return the SDMA engine error interrupt name.
2009 */
2010static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
2011{
2012 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
2013 return buf;
2014}
2015
2016/*
2017 * Return the send context error interrupt name.
2018 */
2019static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
2020{
2021 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
2022 return buf;
2023}
2024
2025static const char * const various_names[] = {
2026 "PbcInt",
2027 "GpioAssertInt",
2028 "Qsfp1Int",
2029 "Qsfp2Int",
2030 "TCritInt"
2031};
2032
2033/*
2034 * Return the various interrupt name.
2035 */
2036static char *is_various_name(char *buf, size_t bsize, unsigned int source)
2037{
2038 if (source < ARRAY_SIZE(various_names))
2039 strncpy(buf, various_names[source], bsize);
2040 else
2041 snprintf(buf, bsize, "Reserved%u", source+IS_VARIOUS_START);
2042 return buf;
2043}
2044
2045/*
2046 * Return the DC interrupt name.
2047 */
2048static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
2049{
2050 static const char * const dc_int_names[] = {
2051 "common",
2052 "lcb",
2053 "8051",
2054 "lbm" /* local block merge */
2055 };
2056
2057 if (source < ARRAY_SIZE(dc_int_names))
2058 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
2059 else
2060 snprintf(buf, bsize, "DCInt%u", source);
2061 return buf;
2062}
2063
2064static const char * const sdma_int_names[] = {
2065 "SDmaInt",
2066 "SdmaIdleInt",
2067 "SdmaProgressInt",
2068};
2069
2070/*
2071 * Return the SDMA engine interrupt name.
2072 */
2073static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
2074{
2075 /* what interrupt */
2076 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
2077 /* which engine */
2078 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
2079
2080 if (likely(what < 3))
2081 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
2082 else
2083 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
2084 return buf;
2085}
2086
2087/*
2088 * Return the receive available interrupt name.
2089 */
2090static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
2091{
2092 snprintf(buf, bsize, "RcvAvailInt%u", source);
2093 return buf;
2094}
2095
2096/*
2097 * Return the receive urgent interrupt name.
2098 */
2099static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
2100{
2101 snprintf(buf, bsize, "RcvUrgentInt%u", source);
2102 return buf;
2103}
2104
2105/*
2106 * Return the send credit interrupt name.
2107 */
2108static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
2109{
2110 snprintf(buf, bsize, "SendCreditInt%u", source);
2111 return buf;
2112}
2113
2114/*
2115 * Return the reserved interrupt name.
2116 */
2117static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
2118{
2119 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
2120 return buf;
2121}
2122
2123static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
2124{
2125 return flag_string(buf, buf_len, flags,
2126 cce_err_status_flags, ARRAY_SIZE(cce_err_status_flags));
2127}
2128
2129static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
2130{
2131 return flag_string(buf, buf_len, flags,
2132 rxe_err_status_flags, ARRAY_SIZE(rxe_err_status_flags));
2133}
2134
2135static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
2136{
2137 return flag_string(buf, buf_len, flags, misc_err_status_flags,
2138 ARRAY_SIZE(misc_err_status_flags));
2139}
2140
2141static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
2142{
2143 return flag_string(buf, buf_len, flags,
2144 pio_err_status_flags, ARRAY_SIZE(pio_err_status_flags));
2145}
2146
2147static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
2148{
2149 return flag_string(buf, buf_len, flags,
2150 sdma_err_status_flags,
2151 ARRAY_SIZE(sdma_err_status_flags));
2152}
2153
2154static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
2155{
2156 return flag_string(buf, buf_len, flags,
2157 egress_err_status_flags, ARRAY_SIZE(egress_err_status_flags));
2158}
2159
2160static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
2161{
2162 return flag_string(buf, buf_len, flags,
2163 egress_err_info_flags, ARRAY_SIZE(egress_err_info_flags));
2164}
2165
2166static char *send_err_status_string(char *buf, int buf_len, u64 flags)
2167{
2168 return flag_string(buf, buf_len, flags,
2169 send_err_status_flags,
2170 ARRAY_SIZE(send_err_status_flags));
2171}
2172
2173static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2174{
2175 char buf[96];
2176
2177 /*
2178 * For most these errors, there is nothing that can be done except
2179 * report or record it.
2180 */
2181 dd_dev_info(dd, "CCE Error: %s\n",
2182 cce_err_status_string(buf, sizeof(buf), reg));
2183
995deafa
MM		 2184 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
2185 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 2186 /* this error requires a manual drop into SPC freeze mode */
2187 /* then a fix up */
2188 start_freeze_handling(dd->pport, FREEZE_SELF);
2189 }
2190}
2191
2192/*
2193 * Check counters for receive errors that do not have an interrupt
2194 * associated with them.
2195 */
2196#define RCVERR_CHECK_TIME 10
2197static void update_rcverr_timer(unsigned long opaque)
2198{
2199 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
2200 struct hfi1_pportdata *ppd = dd->pport;
2201 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
2202
2203 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
2204 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
2205 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
2206 set_link_down_reason(ppd,
2207 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
2208 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
2209 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
2210 }
2211 dd->rcv_ovfl_cnt = (u32) cur_ovfl_cnt;
2212
2213 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
2214}
2215
2216static int init_rcverr(struct hfi1_devdata *dd)
2217{
24523a94 2218 setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
77241056
MM		 2219 /* Assume the hardware counter has been reset */
2220 dd->rcv_ovfl_cnt = 0;
2221 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
2222}
2223
2224static void free_rcverr(struct hfi1_devdata *dd)
2225{
2226 if (dd->rcverr_timer.data)
2227 del_timer_sync(&dd->rcverr_timer);
2228 dd->rcverr_timer.data = 0;
2229}
2230
2231static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2232{
2233 char buf[96];
2234
2235 dd_dev_info(dd, "Receive Error: %s\n",
2236 rxe_err_status_string(buf, sizeof(buf), reg));
2237
2238 if (reg & ALL_RXE_FREEZE_ERR) {
2239 int flags = 0;
2240
2241 /*
2242 * Freeze mode recovery is disabled for the errors
2243 * in RXE_FREEZE_ABORT_MASK
2244 */
995deafa 2245 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
77241056
MM		 2246 flags = FREEZE_ABORT;
2247
2248 start_freeze_handling(dd->pport, flags);
2249 }
2250}
2251
2252static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2253{
2254 char buf[96];
2255
2256 dd_dev_info(dd, "Misc Error: %s",
2257 misc_err_status_string(buf, sizeof(buf), reg));
2258}
2259
2260static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2261{
2262 char buf[96];
2263
2264 dd_dev_info(dd, "PIO Error: %s\n",
2265 pio_err_status_string(buf, sizeof(buf), reg));
2266
2267 if (reg & ALL_PIO_FREEZE_ERR)
2268 start_freeze_handling(dd->pport, 0);
2269}
2270
2271static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2272{
2273 char buf[96];
2274
2275 dd_dev_info(dd, "SDMA Error: %s\n",
2276 sdma_err_status_string(buf, sizeof(buf), reg));
2277
2278 if (reg & ALL_SDMA_FREEZE_ERR)
2279 start_freeze_handling(dd->pport, 0);
2280}
2281
2282static void count_port_inactive(struct hfi1_devdata *dd)
2283{
2284 struct hfi1_pportdata *ppd = dd->pport;
2285
2286 if (ppd->port_xmit_discards < ~(u64)0)
2287 ppd->port_xmit_discards++;
2288}
2289
2290/*
2291 * We have had a "disallowed packet" error during egress. Determine the
2292 * integrity check which failed, and update relevant error counter, etc.
2293 *
2294 * Note that the SEND_EGRESS_ERR_INFO register has only a single
2295 * bit of state per integrity check, and so we can miss the reason for an
2296 * egress error if more than one packet fails the same integrity check
2297 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
2298 */
2299static void handle_send_egress_err_info(struct hfi1_devdata *dd)
2300{
2301 struct hfi1_pportdata *ppd = dd->pport;
2302 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
2303 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
2304 char buf[96];
2305
2306 /* clear down all observed info as quickly as possible after read */
2307 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
2308
2309 dd_dev_info(dd,
2310 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
2311 info, egress_err_info_string(buf, sizeof(buf), info), src);
2312
2313 /* Eventually add other counters for each bit */
2314
2315 if (info & SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK) {
2316 if (ppd->port_xmit_discards < ~(u64)0)
2317 ppd->port_xmit_discards++;
2318 }
2319}
2320
2321/*
2322 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
2323 * register. Does it represent a 'port inactive' error?
2324 */
2325static inline int port_inactive_err(u64 posn)
2326{
2327 return (posn >= SEES(TX_LINKDOWN) &&
2328 posn <= SEES(TX_INCORRECT_LINK_STATE));
2329}
2330
2331/*
2332 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
2333 * register. Does it represent a 'disallowed packet' error?
2334 */
2335static inline int disallowed_pkt_err(u64 posn)
2336{
2337 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
2338 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
2339}
2340
2341static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2342{
2343 u64 reg_copy = reg, handled = 0;
2344 char buf[96];
2345
2346 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
2347 start_freeze_handling(dd->pport, 0);
995deafa 2348 if (is_ax(dd) && (reg &
77241056
MM		 2349 SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK)
2350 && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
2351 start_freeze_handling(dd->pport, 0);
2352
2353 while (reg_copy) {
2354 int posn = fls64(reg_copy);
2355 /*
2356 * fls64() returns a 1-based offset, but we generally
2357 * want 0-based offsets.
2358 */
2359 int shift = posn - 1;
2360
2361 if (port_inactive_err(shift)) {
2362 count_port_inactive(dd);
2363 handled |= (1ULL << shift);
2364 } else if (disallowed_pkt_err(shift)) {
2365 handle_send_egress_err_info(dd);
2366 handled |= (1ULL << shift);
2367 }
2368 clear_bit(shift, (unsigned long *)&reg_copy);
2369 }
2370
2371 reg &= ~handled;
2372
2373 if (reg)
2374 dd_dev_info(dd, "Egress Error: %s\n",
2375 egress_err_status_string(buf, sizeof(buf), reg));
2376}
2377
2378static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2379{
2380 char buf[96];
2381
2382 dd_dev_info(dd, "Send Error: %s\n",
2383 send_err_status_string(buf, sizeof(buf), reg));
2384
2385}
2386
2387/*
2388 * The maximum number of times the error clear down will loop before
2389 * blocking a repeating error. This value is arbitrary.
2390 */
2391#define MAX_CLEAR_COUNT 20
2392
2393/*
2394 * Clear and handle an error register. All error interrupts are funneled
2395 * through here to have a central location to correctly handle single-
2396 * or multi-shot errors.
2397 *
2398 * For non per-context registers, call this routine with a context value
2399 * of 0 so the per-context offset is zero.
2400 *
2401 * If the handler loops too many times, assume that something is wrong
2402 * and can't be fixed, so mask the error bits.
2403 */
2404static void interrupt_clear_down(struct hfi1_devdata *dd,
2405 u32 context,
2406 const struct err_reg_info *eri)
2407{
2408 u64 reg;
2409 u32 count;
2410
2411 /* read in a loop until no more errors are seen */
2412 count = 0;
2413 while (1) {
2414 reg = read_kctxt_csr(dd, context, eri->status);
2415 if (reg == 0)
2416 break;
2417 write_kctxt_csr(dd, context, eri->clear, reg);
2418 if (likely(eri->handler))
2419 eri->handler(dd, context, reg);
2420 count++;
2421 if (count > MAX_CLEAR_COUNT) {
2422 u64 mask;
2423
2424 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
2425 eri->desc, reg);
2426 /*
2427 * Read-modify-write so any other masked bits
2428 * remain masked.
2429 */
2430 mask = read_kctxt_csr(dd, context, eri->mask);
2431 mask &= ~reg;
2432 write_kctxt_csr(dd, context, eri->mask, mask);
2433 break;
2434 }
2435 }
2436}
2437
2438/*
2439 * CCE block "misc" interrupt. Source is < 16.
2440 */
2441static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
2442{
2443 const struct err_reg_info *eri = &misc_errs[source];
2444
2445 if (eri->handler) {
2446 interrupt_clear_down(dd, 0, eri);
2447 } else {
2448 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
2449 source);
2450 }
2451}
2452
2453static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
2454{
2455 return flag_string(buf, buf_len, flags,
2456 sc_err_status_flags, ARRAY_SIZE(sc_err_status_flags));
2457}
2458
2459/*
2460 * Send context error interrupt. Source (hw_context) is < 160.
2461 *
2462 * All send context errors cause the send context to halt. The normal
2463 * clear-down mechanism cannot be used because we cannot clear the
2464 * error bits until several other long-running items are done first.
2465 * This is OK because with the context halted, nothing else is going
2466 * to happen on it anyway.
2467 */
2468static void is_sendctxt_err_int(struct hfi1_devdata *dd,
2469 unsigned int hw_context)
2470{
2471 struct send_context_info *sci;
2472 struct send_context *sc;
2473 char flags[96];
2474 u64 status;
2475 u32 sw_index;
2476
2477 sw_index = dd->hw_to_sw[hw_context];
2478 if (sw_index >= dd->num_send_contexts) {
2479 dd_dev_err(dd,
2480 "out of range sw index %u for send context %u\n",
2481 sw_index, hw_context);
2482 return;
2483 }
2484 sci = &dd->send_contexts[sw_index];
2485 sc = sci->sc;
2486 if (!sc) {
2487 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
2488 sw_index, hw_context);
2489 return;
2490 }
2491
2492 /* tell the software that a halt has begun */
2493 sc_stop(sc, SCF_HALTED);
2494
2495 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
2496
2497 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
2498 send_context_err_status_string(flags, sizeof(flags), status));
2499
2500 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
2501 handle_send_egress_err_info(dd);
2502
2503 /*
2504 * Automatically restart halted kernel contexts out of interrupt
2505 * context. User contexts must ask the driver to restart the context.
2506 */
2507 if (sc->type != SC_USER)
2508 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
2509}
2510
2511static void handle_sdma_eng_err(struct hfi1_devdata *dd,
2512 unsigned int source, u64 status)
2513{
2514 struct sdma_engine *sde;
2515
2516 sde = &dd->per_sdma[source];
2517#ifdef CONFIG_SDMA_VERBOSITY
2518 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
2519 slashstrip(__FILE__), __LINE__, __func__);
2520 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
2521 sde->this_idx, source, (unsigned long long)status);
2522#endif
2523 sdma_engine_error(sde, status);
2524}
2525
2526/*
2527 * CCE block SDMA error interrupt. Source is < 16.
2528 */
2529static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
2530{
2531#ifdef CONFIG_SDMA_VERBOSITY
2532 struct sdma_engine *sde = &dd->per_sdma[source];
2533
2534 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
2535 slashstrip(__FILE__), __LINE__, __func__);
2536 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
2537 source);
2538 sdma_dumpstate(sde);
2539#endif
2540 interrupt_clear_down(dd, source, &sdma_eng_err);
2541}
2542
2543/*
2544 * CCE block "various" interrupt. Source is < 8.
2545 */
2546static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
2547{
2548 const struct err_reg_info *eri = &various_err[source];
2549
2550 /*
2551 * TCritInt cannot go through interrupt_clear_down()
2552 * because it is not a second tier interrupt. The handler
2553 * should be called directly.
2554 */
2555 if (source == TCRIT_INT_SOURCE)
2556 handle_temp_err(dd);
2557 else if (eri->handler)
2558 interrupt_clear_down(dd, 0, eri);
2559 else
2560 dd_dev_info(dd,
2561 "%s: Unimplemented/reserved interrupt %d\n",
2562 __func__, source);
2563}
2564
2565static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
2566{
2567 /* source is always zero */
2568 struct hfi1_pportdata *ppd = dd->pport;
2569 unsigned long flags;
2570 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
2571
2572 if (reg & QSFP_HFI0_MODPRST_N) {
2573
2574 dd_dev_info(dd, "%s: ModPresent triggered QSFP interrupt\n",
2575 __func__);
2576
2577 if (!qsfp_mod_present(ppd)) {
2578 ppd->driver_link_ready = 0;
2579 /*
2580 * Cable removed, reset all our information about the
2581 * cache and cable capabilities
2582 */
2583
2584 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2585 /*
2586 * We don't set cache_refresh_required here as we expect
2587 * an interrupt when a cable is inserted
2588 */
2589 ppd->qsfp_info.cache_valid = 0;
2590 ppd->qsfp_info.qsfp_interrupt_functional = 0;
2591 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
2592 flags);
2593 write_csr(dd,
2594 dd->hfi1_id ?
2595 ASIC_QSFP2_INVERT :
2596 ASIC_QSFP1_INVERT,
2597 qsfp_int_mgmt);
2598 if (ppd->host_link_state == HLS_DN_POLL) {
2599 /*
2600 * The link is still in POLL. This means
2601 * that the normal link down processing
2602 * will not happen. We have to do it here
2603 * before turning the DC off.
2604 */
2605 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
2606 }
2607 } else {
2608 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2609 ppd->qsfp_info.cache_valid = 0;
2610 ppd->qsfp_info.cache_refresh_required = 1;
2611 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
2612 flags);
2613
2614 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
2615 write_csr(dd,
2616 dd->hfi1_id ?
2617 ASIC_QSFP2_INVERT :
2618 ASIC_QSFP1_INVERT,
2619 qsfp_int_mgmt);
2620 }
2621 }
2622
2623 if (reg & QSFP_HFI0_INT_N) {
2624
2625 dd_dev_info(dd, "%s: IntN triggered QSFP interrupt\n",
2626 __func__);
2627 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2628 ppd->qsfp_info.check_interrupt_flags = 1;
2629 ppd->qsfp_info.qsfp_interrupt_functional = 1;
2630 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
2631 }
2632
2633 /* Schedule the QSFP work only if there is a cable attached. */
2634 if (qsfp_mod_present(ppd))
2635 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
2636}
2637
2638static int request_host_lcb_access(struct hfi1_devdata *dd)
2639{
2640 int ret;
2641
2642 ret = do_8051_command(dd, HCMD_MISC,
2643 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << LOAD_DATA_FIELD_ID_SHIFT,
2644 NULL);
2645 if (ret != HCMD_SUCCESS) {
2646 dd_dev_err(dd, "%s: command failed with error %d\n",
2647 __func__, ret);
2648 }
2649 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
2650}
2651
2652static int request_8051_lcb_access(struct hfi1_devdata *dd)
2653{
2654 int ret;
2655
2656 ret = do_8051_command(dd, HCMD_MISC,
2657 (u64)HCMD_MISC_GRANT_LCB_ACCESS << LOAD_DATA_FIELD_ID_SHIFT,
2658 NULL);
2659 if (ret != HCMD_SUCCESS) {
2660 dd_dev_err(dd, "%s: command failed with error %d\n",
2661 __func__, ret);
2662 }
2663 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
2664}
2665
2666/*
2667 * Set the LCB selector - allow host access. The DCC selector always
2668 * points to the host.
2669 */
2670static inline void set_host_lcb_access(struct hfi1_devdata *dd)
2671{
2672 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
2673 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK
2674 | DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
2675}
2676
2677/*
2678 * Clear the LCB selector - allow 8051 access. The DCC selector always
2679 * points to the host.
2680 */
2681static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
2682{
2683 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
2684 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
2685}
2686
2687/*
2688 * Acquire LCB access from the 8051. If the host already has access,
2689 * just increment a counter. Otherwise, inform the 8051 that the
2690 * host is taking access.
2691 *
2692 * Returns:
2693 * 0 on success
2694 * -EBUSY if the 8051 has control and cannot be disturbed
2695 * -errno if unable to acquire access from the 8051
2696 */
2697int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
2698{
2699 struct hfi1_pportdata *ppd = dd->pport;
2700 int ret = 0;
2701
2702 /*
2703 * Use the host link state lock so the operation of this routine
2704 * { link state check, selector change, count increment } can occur
2705 * as a unit against a link state change. Otherwise there is a
2706 * race between the state change and the count increment.
2707 */
2708 if (sleep_ok) {
2709 mutex_lock(&ppd->hls_lock);
2710 } else {
951842b0 2711 while (!mutex_trylock(&ppd->hls_lock))
77241056
MM		 2712 udelay(1);
2713 }
2714
2715 /* this access is valid only when the link is up */
2716 if ((ppd->host_link_state & HLS_UP) == 0) {
2717 dd_dev_info(dd, "%s: link state %s not up\n",
2718 __func__, link_state_name(ppd->host_link_state));
2719 ret = -EBUSY;
2720 goto done;
2721 }
2722
2723 if (dd->lcb_access_count == 0) {
2724 ret = request_host_lcb_access(dd);
2725 if (ret) {
2726 dd_dev_err(dd,
2727 "%s: unable to acquire LCB access, err %d\n",
2728 __func__, ret);
2729 goto done;
2730 }
2731 set_host_lcb_access(dd);
2732 }
2733 dd->lcb_access_count++;
2734done:
2735 mutex_unlock(&ppd->hls_lock);
2736 return ret;
2737}
2738
2739/*
2740 * Release LCB access by decrementing the use count. If the count is moving
2741 * from 1 to 0, inform 8051 that it has control back.
2742 *
2743 * Returns:
2744 * 0 on success
2745 * -errno if unable to release access to the 8051
2746 */
2747int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
2748{
2749 int ret = 0;
2750
2751 /*
2752 * Use the host link state lock because the acquire needed it.
2753 * Here, we only need to keep { selector change, count decrement }
2754 * as a unit.
2755 */
2756 if (sleep_ok) {
2757 mutex_lock(&dd->pport->hls_lock);
2758 } else {
951842b0 2759 while (!mutex_trylock(&dd->pport->hls_lock))
77241056
MM		 2760 udelay(1);
2761 }
2762
2763 if (dd->lcb_access_count == 0) {
2764 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
2765 __func__);
2766 goto done;
2767 }
2768
2769 if (dd->lcb_access_count == 1) {
2770 set_8051_lcb_access(dd);
2771 ret = request_8051_lcb_access(dd);
2772 if (ret) {
2773 dd_dev_err(dd,
2774 "%s: unable to release LCB access, err %d\n",
2775 __func__, ret);
2776 /* restore host access if the grant didn't work */
2777 set_host_lcb_access(dd);
2778 goto done;
2779 }
2780 }
2781 dd->lcb_access_count--;
2782done:
2783 mutex_unlock(&dd->pport->hls_lock);
2784 return ret;
2785}
2786
2787/*
2788 * Initialize LCB access variables and state. Called during driver load,
2789 * after most of the initialization is finished.
2790 *
2791 * The DC default is LCB access on for the host. The driver defaults to
2792 * leaving access to the 8051. Assign access now - this constrains the call
2793 * to this routine to be after all LCB set-up is done. In particular, after
2794 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
2795 */
2796static void init_lcb_access(struct hfi1_devdata *dd)
2797{
2798 dd->lcb_access_count = 0;
2799}
2800
2801/*
2802 * Write a response back to a 8051 request.
2803 */
2804static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
2805{
2806 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
2807 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK
2808 | (u64)return_code << DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT
2809 | (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
2810}
2811
2812/*
2813 * Handle requests from the 8051.
2814 */
2815static void handle_8051_request(struct hfi1_devdata *dd)
2816{
2817 u64 reg;
2818 u16 data;
2819 u8 type;
2820
2821 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
2822 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
2823 return; /* no request */
2824
2825 /* zero out COMPLETED so the response is seen */
2826 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
2827
2828 /* extract request details */
2829 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
2830 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
2831 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
2832 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
2833
2834 switch (type) {
2835 case HREQ_LOAD_CONFIG:
2836 case HREQ_SAVE_CONFIG:
2837 case HREQ_READ_CONFIG:
2838 case HREQ_SET_TX_EQ_ABS:
2839 case HREQ_SET_TX_EQ_REL:
2840 case HREQ_ENABLE:
2841 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
2842 type);
2843 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
2844 break;
2845
2846 case HREQ_CONFIG_DONE:
2847 hreq_response(dd, HREQ_SUCCESS, 0);
2848 break;
2849
2850 case HREQ_INTERFACE_TEST:
2851 hreq_response(dd, HREQ_SUCCESS, data);
2852 break;
2853
2854 default:
2855 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
2856 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
2857 break;
2858 }
2859}
2860
2861static void write_global_credit(struct hfi1_devdata *dd,
2862 u8 vau, u16 total, u16 shared)
2863{
2864 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
2865 ((u64)total
2866 << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
2867 | ((u64)shared
2868 << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
2869 | ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
2870}
2871
2872/*
2873 * Set up initial VL15 credits of the remote. Assumes the rest of
2874 * the CM credit registers are zero from a previous global or credit reset .
2875 */
2876void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
2877{
2878 /* leave shared count at zero for both global and VL15 */
2879 write_global_credit(dd, vau, vl15buf, 0);
2880
2881 /* We may need some credits for another VL when sending packets
2882 * with the snoop interface. Dividing it down the middle for VL15
2883 * and VL0 should suffice.
2884 */
2885 if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) {
2886 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1)
2887 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
2888 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1)
2889 << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT);
2890 } else {
2891 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
2892 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
2893 }
2894}
2895
2896/*
2897 * Zero all credit details from the previous connection and
2898 * reset the CM manager's internal counters.
2899 */
2900void reset_link_credits(struct hfi1_devdata *dd)
2901{
2902 int i;
2903
2904 /* remove all previous VL credit limits */
2905 for (i = 0; i < TXE_NUM_DATA_VL; i++)
2906 write_csr(dd, SEND_CM_CREDIT_VL + (8*i), 0);
2907 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
2908 write_global_credit(dd, 0, 0, 0);
2909 /* reset the CM block */
2910 pio_send_control(dd, PSC_CM_RESET);
2911}
2912
2913/* convert a vCU to a CU */
2914static u32 vcu_to_cu(u8 vcu)
2915{
2916 return 1 << vcu;
2917}
2918
2919/* convert a CU to a vCU */
2920static u8 cu_to_vcu(u32 cu)
2921{
2922 return ilog2(cu);
2923}
2924
2925/* convert a vAU to an AU */
2926static u32 vau_to_au(u8 vau)
2927{
2928 return 8 * (1 << vau);
2929}
2930
2931static void set_linkup_defaults(struct hfi1_pportdata *ppd)
2932{
2933 ppd->sm_trap_qp = 0x0;
2934 ppd->sa_qp = 0x1;
2935}
2936
2937/*
2938 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
2939 */
2940static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
2941{
2942 u64 reg;
2943
2944 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
2945 write_csr(dd, DC_LCB_CFG_RUN, 0);
2946 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
2947 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
2948 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
2949 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
2950 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
2951 reg = read_csr(dd, DCC_CFG_RESET);
2952 write_csr(dd, DCC_CFG_RESET,
2953 reg
2954 | (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT)
2955 | (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
2956 (void) read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
2957 if (!abort) {
2958 udelay(1); /* must hold for the longer of 16cclks or 20ns */
2959 write_csr(dd, DCC_CFG_RESET, reg);
2960 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
2961 }
2962}
2963
2964/*
2965 * This routine should be called after the link has been transitioned to
2966 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
2967 * reset).
2968 *
2969 * The expectation is that the caller of this routine would have taken
2970 * care of properly transitioning the link into the correct state.
2971 */
2972static void dc_shutdown(struct hfi1_devdata *dd)
2973{
2974 unsigned long flags;
2975
2976 spin_lock_irqsave(&dd->dc8051_lock, flags);
2977 if (dd->dc_shutdown) {
2978 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
2979 return;
2980 }
2981 dd->dc_shutdown = 1;
2982 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
2983 /* Shutdown the LCB */
2984 lcb_shutdown(dd, 1);
2985 /* Going to OFFLINE would have causes the 8051 to put the
2986 * SerDes into reset already. Just need to shut down the 8051,
2987 * itself. */
2988 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
2989}
2990
2991/* Calling this after the DC has been brought out of reset should not
2992 * do any damage. */
2993static void dc_start(struct hfi1_devdata *dd)
2994{
2995 unsigned long flags;
2996 int ret;
2997
2998 spin_lock_irqsave(&dd->dc8051_lock, flags);
2999 if (!dd->dc_shutdown)
3000 goto done;
3001 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
3002 /* Take the 8051 out of reset */
3003 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
3004 /* Wait until 8051 is ready */
3005 ret = wait_fm_ready(dd, TIMEOUT_8051_START);
3006 if (ret) {
3007 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
3008 __func__);
3009 }
3010 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
3011 write_csr(dd, DCC_CFG_RESET, 0x10);
3012 /* lcb_shutdown() with abort=1 does not restore these */
3013 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
3014 spin_lock_irqsave(&dd->dc8051_lock, flags);
3015 dd->dc_shutdown = 0;
3016done:
3017 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
3018}
3019
3020/*
3021 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
3022 */
3023static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
3024{
3025 u64 rx_radr, tx_radr;
3026 u32 version;
3027
3028 if (dd->icode != ICODE_FPGA_EMULATION)
3029 return;
3030
3031 /*
3032 * These LCB defaults on emulator _s are good, nothing to do here:
3033 * LCB_CFG_TX_FIFOS_RADR
3034 * LCB_CFG_RX_FIFOS_RADR
3035 * LCB_CFG_LN_DCLK
3036 * LCB_CFG_IGNORE_LOST_RCLK
3037 */
3038 if (is_emulator_s(dd))
3039 return;
3040 /* else this is _p */
3041
3042 version = emulator_rev(dd);
995deafa 3043 if (!is_ax(dd))
77241056
MM		 3044 version = 0x2d; /* all B0 use 0x2d or higher settings */
3045
3046 if (version <= 0x12) {
3047 /* release 0x12 and below */
3048
3049 /*
3050 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
3051 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
3052 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
3053 */
3054 rx_radr =
3055 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3056 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3057 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3058 /*
3059 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
3060 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
3061 */
3062 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3063 } else if (version <= 0x18) {
3064 /* release 0x13 up to 0x18 */
3065 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
3066 rx_radr =
3067 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3068 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3069 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3070 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3071 } else if (version == 0x19) {
3072 /* release 0x19 */
3073 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
3074 rx_radr =
3075 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3076 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3077 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3078 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3079 } else if (version == 0x1a) {
3080 /* release 0x1a */
3081 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
3082 rx_radr =
3083 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3084 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3085 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3086 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3087 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
3088 } else {
3089 /* release 0x1b and higher */
3090 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
3091 rx_radr =
3092 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3093 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3094 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3095 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3096 }
3097
3098 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
3099 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
3100 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
3101 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
3102 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
3103}
3104
3105/*
3106 * Handle a SMA idle message
3107 *
3108 * This is a work-queue function outside of the interrupt.
3109 */
3110void handle_sma_message(struct work_struct *work)
3111{
3112 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3113 sma_message_work);
3114 struct hfi1_devdata *dd = ppd->dd;
3115 u64 msg;
3116 int ret;
3117
3118 /* msg is bytes 1-4 of the 40-bit idle message - the command code
3119 is stripped off */
3120 ret = read_idle_sma(dd, &msg);
3121 if (ret)
3122 return;
3123 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
3124 /*
3125 * React to the SMA message. Byte[1] (0 for us) is the command.
3126 */
3127 switch (msg & 0xff) {
3128 case SMA_IDLE_ARM:
3129 /*
3130 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
3131 * State Transitions
3132 *
3133 * Only expected in INIT or ARMED, discard otherwise.
3134 */
3135 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
3136 ppd->neighbor_normal = 1;
3137 break;
3138 case SMA_IDLE_ACTIVE:
3139 /*
3140 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
3141 * State Transitions
3142 *
3143 * Can activate the node. Discard otherwise.
3144 */
3145 if (ppd->host_link_state == HLS_UP_ARMED
3146 && ppd->is_active_optimize_enabled) {
3147 ppd->neighbor_normal = 1;
3148 ret = set_link_state(ppd, HLS_UP_ACTIVE);
3149 if (ret)
3150 dd_dev_err(
3151 dd,
3152 "%s: received Active SMA idle message, couldn't set link to Active\n",
3153 __func__);
3154 }
3155 break;
3156 default:
3157 dd_dev_err(dd,
3158 "%s: received unexpected SMA idle message 0x%llx\n",
3159 __func__, msg);
3160 break;
3161 }
3162}
3163
3164static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
3165{
3166 u64 rcvctrl;
3167 unsigned long flags;
3168
3169 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
3170 rcvctrl = read_csr(dd, RCV_CTRL);
3171 rcvctrl |= add;
3172 rcvctrl &= ~clear;
3173 write_csr(dd, RCV_CTRL, rcvctrl);
3174 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
3175}
3176
3177static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
3178{
3179 adjust_rcvctrl(dd, add, 0);
3180}
3181
3182static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
3183{
3184 adjust_rcvctrl(dd, 0, clear);
3185}
3186
3187/*
3188 * Called from all interrupt handlers to start handling an SPC freeze.
3189 */
3190void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
3191{
3192 struct hfi1_devdata *dd = ppd->dd;
3193 struct send_context *sc;
3194 int i;
3195
3196 if (flags & FREEZE_SELF)
3197 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
3198
3199 /* enter frozen mode */
3200 dd->flags |= HFI1_FROZEN;
3201
3202 /* notify all SDMA engines that they are going into a freeze */
3203 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
3204
3205 /* do halt pre-handling on all enabled send contexts */
3206 for (i = 0; i < dd->num_send_contexts; i++) {
3207 sc = dd->send_contexts[i].sc;
3208 if (sc && (sc->flags & SCF_ENABLED))
3209 sc_stop(sc, SCF_FROZEN | SCF_HALTED);
3210 }
3211
3212 /* Send context are frozen. Notify user space */
3213 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
3214
3215 if (flags & FREEZE_ABORT) {
3216 dd_dev_err(dd,
3217 "Aborted freeze recovery. Please REBOOT system\n");
3218 return;
3219 }
3220 /* queue non-interrupt handler */
3221 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
3222}
3223
3224/*
3225 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
3226 * depending on the "freeze" parameter.
3227 *
3228 * No need to return an error if it times out, our only option
3229 * is to proceed anyway.
3230 */
3231static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
3232{
3233 unsigned long timeout;
3234 u64 reg;
3235
3236 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
3237 while (1) {
3238 reg = read_csr(dd, CCE_STATUS);
3239 if (freeze) {
3240 /* waiting until all indicators are set */
3241 if ((reg & ALL_FROZE) == ALL_FROZE)
3242 return; /* all done */
3243 } else {
3244 /* waiting until all indicators are clear */
3245 if ((reg & ALL_FROZE) == 0)
3246 return; /* all done */
3247 }
3248
3249 if (time_after(jiffies, timeout)) {
3250 dd_dev_err(dd,
3251 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
3252 freeze ? "" : "un",
3253 reg & ALL_FROZE,
3254 freeze ? ALL_FROZE : 0ull);
3255 return;
3256 }
3257 usleep_range(80, 120);
3258 }
3259}
3260
3261/*
3262 * Do all freeze handling for the RXE block.
3263 */
3264static void rxe_freeze(struct hfi1_devdata *dd)
3265{
3266 int i;
3267
3268 /* disable port */
3269 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3270
3271 /* disable all receive contexts */
3272 for (i = 0; i < dd->num_rcv_contexts; i++)
3273 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
3274}
3275
3276/*
3277 * Unfreeze handling for the RXE block - kernel contexts only.
3278 * This will also enable the port. User contexts will do unfreeze
3279 * handling on a per-context basis as they call into the driver.
3280 *
3281 */
3282static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
3283{
3284 int i;
3285
3286 /* enable all kernel contexts */
3287 for (i = 0; i < dd->n_krcv_queues; i++)
3288 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB, i);
3289
3290 /* enable port */
3291 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3292}
3293
3294/*
3295 * Non-interrupt SPC freeze handling.
3296 *
3297 * This is a work-queue function outside of the triggering interrupt.
3298 */
3299void handle_freeze(struct work_struct *work)
3300{
3301 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3302 freeze_work);
3303 struct hfi1_devdata *dd = ppd->dd;
3304
3305 /* wait for freeze indicators on all affected blocks */
3306 dd_dev_info(dd, "Entering SPC freeze\n");
3307 wait_for_freeze_status(dd, 1);
3308
3309 /* SPC is now frozen */
3310
3311 /* do send PIO freeze steps */
3312 pio_freeze(dd);
3313
3314 /* do send DMA freeze steps */
3315 sdma_freeze(dd);
3316
3317 /* do send egress freeze steps - nothing to do */
3318
3319 /* do receive freeze steps */
3320 rxe_freeze(dd);
3321
3322 /*
3323 * Unfreeze the hardware - clear the freeze, wait for each
3324 * block's frozen bit to clear, then clear the frozen flag.
3325 */
3326 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
3327 wait_for_freeze_status(dd, 0);
3328
995deafa 3329 if (is_ax(dd)) {
77241056
MM		 3330 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
3331 wait_for_freeze_status(dd, 1);
3332 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
3333 wait_for_freeze_status(dd, 0);
3334 }
3335
3336 /* do send PIO unfreeze steps for kernel contexts */
3337 pio_kernel_unfreeze(dd);
3338
3339 /* do send DMA unfreeze steps */
3340 sdma_unfreeze(dd);
3341
3342 /* do send egress unfreeze steps - nothing to do */
3343
3344 /* do receive unfreeze steps for kernel contexts */
3345 rxe_kernel_unfreeze(dd);
3346
3347 /*
3348 * The unfreeze procedure touches global device registers when
3349 * it disables and re-enables RXE. Mark the device unfrozen
3350 * after all that is done so other parts of the driver waiting
3351 * for the device to unfreeze don't do things out of order.
3352 *
3353 * The above implies that the meaning of HFI1_FROZEN flag is
3354 * "Device has gone into freeze mode and freeze mode handling
3355 * is still in progress."
3356 *
3357 * The flag will be removed when freeze mode processing has
3358 * completed.
3359 */
3360 dd->flags &= ~HFI1_FROZEN;
3361 wake_up(&dd->event_queue);
3362
3363 /* no longer frozen */
3364 dd_dev_err(dd, "Exiting SPC freeze\n");
3365}
3366
3367/*
3368 * Handle a link up interrupt from the 8051.
3369 *
3370 * This is a work-queue function outside of the interrupt.
3371 */
3372void handle_link_up(struct work_struct *work)
3373{
3374 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3375 link_up_work);
3376 set_link_state(ppd, HLS_UP_INIT);
3377
3378 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
3379 read_ltp_rtt(ppd->dd);
3380 /*
3381 * OPA specifies that certain counters are cleared on a transition
3382 * to link up, so do that.
3383 */
3384 clear_linkup_counters(ppd->dd);
3385 /*
3386 * And (re)set link up default values.
3387 */
3388 set_linkup_defaults(ppd);
3389
3390 /* enforce link speed enabled */
3391 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
3392 /* oops - current speed is not enabled, bounce */
3393 dd_dev_err(ppd->dd,
3394 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
3395 ppd->link_speed_active, ppd->link_speed_enabled);
3396 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
3397 OPA_LINKDOWN_REASON_SPEED_POLICY);
3398 set_link_state(ppd, HLS_DN_OFFLINE);
3399 start_link(ppd);
3400 }
3401}
3402
3403/* Several pieces of LNI information were cached for SMA in ppd.
3404 * Reset these on link down */
3405static void reset_neighbor_info(struct hfi1_pportdata *ppd)
3406{
3407 ppd->neighbor_guid = 0;
3408 ppd->neighbor_port_number = 0;
3409 ppd->neighbor_type = 0;
3410 ppd->neighbor_fm_security = 0;
3411}
3412
3413/*
3414 * Handle a link down interrupt from the 8051.
3415 *
3416 * This is a work-queue function outside of the interrupt.
3417 */
3418void handle_link_down(struct work_struct *work)
3419{
3420 u8 lcl_reason, neigh_reason = 0;
3421 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3422 link_down_work);
3423
3424 /* go offline first, then deal with reasons */
3425 set_link_state(ppd, HLS_DN_OFFLINE);
3426
3427 lcl_reason = 0;
3428 read_planned_down_reason_code(ppd->dd, &neigh_reason);
3429
3430 /*
3431 * If no reason, assume peer-initiated but missed
3432 * LinkGoingDown idle flits.
3433 */
3434 if (neigh_reason == 0)
3435 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
3436
3437 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
3438
3439 reset_neighbor_info(ppd);
3440
3441 /* disable the port */
3442 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3443
3444 /* If there is no cable attached, turn the DC off. Otherwise,
3445 * start the link bring up. */
3446 if (!qsfp_mod_present(ppd))
3447 dc_shutdown(ppd->dd);
3448 else
3449 start_link(ppd);
3450}
3451
3452void handle_link_bounce(struct work_struct *work)
3453{
3454 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3455 link_bounce_work);
3456
3457 /*
3458 * Only do something if the link is currently up.
3459 */
3460 if (ppd->host_link_state & HLS_UP) {
3461 set_link_state(ppd, HLS_DN_OFFLINE);
3462 start_link(ppd);
3463 } else {
3464 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
3465 __func__, link_state_name(ppd->host_link_state));
3466 }
3467}
3468
3469/*
3470 * Mask conversion: Capability exchange to Port LTP. The capability
3471 * exchange has an implicit 16b CRC that is mandatory.
3472 */
3473static int cap_to_port_ltp(int cap)
3474{
3475 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
3476
3477 if (cap & CAP_CRC_14B)
3478 port_ltp |= PORT_LTP_CRC_MODE_14;
3479 if (cap & CAP_CRC_48B)
3480 port_ltp |= PORT_LTP_CRC_MODE_48;
3481 if (cap & CAP_CRC_12B_16B_PER_LANE)
3482 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
3483
3484 return port_ltp;
3485}
3486
3487/*
3488 * Convert an OPA Port LTP mask to capability mask
3489 */
3490int port_ltp_to_cap(int port_ltp)
3491{
3492 int cap_mask = 0;
3493
3494 if (port_ltp & PORT_LTP_CRC_MODE_14)
3495 cap_mask |= CAP_CRC_14B;
3496 if (port_ltp & PORT_LTP_CRC_MODE_48)
3497 cap_mask |= CAP_CRC_48B;
3498 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
3499 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
3500
3501 return cap_mask;
3502}
3503
3504/*
3505 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
3506 */
3507static int lcb_to_port_ltp(int lcb_crc)
3508{
3509 int port_ltp = 0;
3510
3511 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
3512 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
3513 else if (lcb_crc == LCB_CRC_48B)
3514 port_ltp = PORT_LTP_CRC_MODE_48;
3515 else if (lcb_crc == LCB_CRC_14B)
3516 port_ltp = PORT_LTP_CRC_MODE_14;
3517 else
3518 port_ltp = PORT_LTP_CRC_MODE_16;
3519
3520 return port_ltp;
3521}
3522
3523/*
3524 * Our neighbor has indicated that we are allowed to act as a fabric
3525 * manager, so place the full management partition key in the second
3526 * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
3527 * that we should already have the limited management partition key in
3528 * array element 1, and also that the port is not yet up when
3529 * add_full_mgmt_pkey() is invoked.
3530 */
3531static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
3532{
3533 struct hfi1_devdata *dd = ppd->dd;
3534
3535 /* Sanity check - ppd->pkeys[2] should be 0 */
3536 if (ppd->pkeys[2] != 0)
3537 dd_dev_err(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
3538 __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
3539 ppd->pkeys[2] = FULL_MGMT_P_KEY;
3540 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
3541}
3542
3543/*
3544 * Convert the given link width to the OPA link width bitmask.
3545 */
3546static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
3547{
3548 switch (width) {
3549 case 0:
3550 /*
3551 * Simulator and quick linkup do not set the width.
3552 * Just set it to 4x without complaint.
3553 */
3554 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
3555 return OPA_LINK_WIDTH_4X;
3556 return 0; /* no lanes up */
3557 case 1: return OPA_LINK_WIDTH_1X;
3558 case 2: return OPA_LINK_WIDTH_2X;
3559 case 3: return OPA_LINK_WIDTH_3X;
3560 default:
3561 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
3562 __func__, width);
3563 /* fall through */
3564 case 4: return OPA_LINK_WIDTH_4X;
3565 }
3566}
3567
3568/*
3569 * Do a population count on the bottom nibble.
3570 */
3571static const u8 bit_counts[16] = {
3572 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
3573};
3574static inline u8 nibble_to_count(u8 nibble)
3575{
3576 return bit_counts[nibble & 0xf];
3577}
3578
3579/*
3580 * Read the active lane information from the 8051 registers and return
3581 * their widths.
3582 *
3583 * Active lane information is found in these 8051 registers:
3584 * enable_lane_tx
3585 * enable_lane_rx
3586 */
3587static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
3588 u16 *rx_width)
3589{
3590 u16 tx, rx;
3591 u8 enable_lane_rx;
3592 u8 enable_lane_tx;
3593 u8 tx_polarity_inversion;
3594 u8 rx_polarity_inversion;
3595 u8 max_rate;
3596
3597 /* read the active lanes */
3598 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
3599 &rx_polarity_inversion, &max_rate);
3600 read_local_lni(dd, &enable_lane_rx);
3601
3602 /* convert to counts */
3603 tx = nibble_to_count(enable_lane_tx);
3604 rx = nibble_to_count(enable_lane_rx);
3605
3606 /*
3607 * Set link_speed_active here, overriding what was set in
3608 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
3609 * set the max_rate field in handle_verify_cap until v0.19.
3610 */
3611 if ((dd->icode == ICODE_RTL_SILICON)
3612 && (dd->dc8051_ver < dc8051_ver(0, 19))) {
3613 /* max_rate: 0 = 12.5G, 1 = 25G */
3614 switch (max_rate) {
3615 case 0:
3616 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
3617 break;
3618 default:
3619 dd_dev_err(dd,
3620 "%s: unexpected max rate %d, using 25Gb\n",
3621 __func__, (int)max_rate);
3622 /* fall through */
3623 case 1:
3624 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
3625 break;
3626 }
3627 }
3628
3629 dd_dev_info(dd,
3630 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
3631 enable_lane_tx, tx, enable_lane_rx, rx);
3632 *tx_width = link_width_to_bits(dd, tx);
3633 *rx_width = link_width_to_bits(dd, rx);
3634}
3635
3636/*
3637 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
3638 * Valid after the end of VerifyCap and during LinkUp. Does not change
3639 * after link up. I.e. look elsewhere for downgrade information.
3640 *
3641 * Bits are:
3642 * + bits [7:4] contain the number of active transmitters
3643 * + bits [3:0] contain the number of active receivers
3644 * These are numbers 1 through 4 and can be different values if the
3645 * link is asymmetric.
3646 *
3647 * verify_cap_local_fm_link_width[0] retains its original value.
3648 */
3649static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
3650 u16 *rx_width)
3651{
3652 u16 widths, tx, rx;
3653 u8 misc_bits, local_flags;
3654 u16 active_tx, active_rx;
3655
3656 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
3657 tx = widths >> 12;
3658 rx = (widths >> 8) & 0xf;
3659
3660 *tx_width = link_width_to_bits(dd, tx);
3661 *rx_width = link_width_to_bits(dd, rx);
3662
3663 /* print the active widths */
3664 get_link_widths(dd, &active_tx, &active_rx);
3665}
3666
3667/*
3668 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
3669 * hardware information when the link first comes up.
3670 *
3671 * The link width is not available until after VerifyCap.AllFramesReceived
3672 * (the trigger for handle_verify_cap), so this is outside that routine
3673 * and should be called when the 8051 signals linkup.
3674 */
3675void get_linkup_link_widths(struct hfi1_pportdata *ppd)
3676{
3677 u16 tx_width, rx_width;
3678
3679 /* get end-of-LNI link widths */
3680 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
3681
3682 /* use tx_width as the link is supposed to be symmetric on link up */
3683 ppd->link_width_active = tx_width;
3684 /* link width downgrade active (LWD.A) starts out matching LW.A */
3685 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
3686 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
3687 /* per OPA spec, on link up LWD.E resets to LWD.S */
3688 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
3689 /* cache the active egress rate (units {10^6 bits/sec]) */
3690 ppd->current_egress_rate = active_egress_rate(ppd);
3691}
3692
3693/*
3694 * Handle a verify capabilities interrupt from the 8051.
3695 *
3696 * This is a work-queue function outside of the interrupt.
3697 */
3698void handle_verify_cap(struct work_struct *work)
3699{
3700 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3701 link_vc_work);
3702 struct hfi1_devdata *dd = ppd->dd;
3703 u64 reg;
3704 u8 power_management;
3705 u8 continious;
3706 u8 vcu;
3707 u8 vau;
3708 u8 z;
3709 u16 vl15buf;
3710 u16 link_widths;
3711 u16 crc_mask;
3712 u16 crc_val;
3713 u16 device_id;
3714 u16 active_tx, active_rx;
3715 u8 partner_supported_crc;
3716 u8 remote_tx_rate;
3717 u8 device_rev;
3718
3719 set_link_state(ppd, HLS_VERIFY_CAP);
3720
3721 lcb_shutdown(dd, 0);
3722 adjust_lcb_for_fpga_serdes(dd);
3723
3724 /*
3725 * These are now valid:
3726 * remote VerifyCap fields in the general LNI config
3727 * CSR DC8051_STS_REMOTE_GUID
3728 * CSR DC8051_STS_REMOTE_NODE_TYPE
3729 * CSR DC8051_STS_REMOTE_FM_SECURITY
3730 * CSR DC8051_STS_REMOTE_PORT_NO
3731 */
3732
3733 read_vc_remote_phy(dd, &power_management, &continious);
3734 read_vc_remote_fabric(
3735 dd,
3736 &vau,
3737 &z,
3738 &vcu,
3739 &vl15buf,
3740 &partner_supported_crc);
3741 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
3742 read_remote_device_id(dd, &device_id, &device_rev);
3743 /*
3744 * And the 'MgmtAllowed' information, which is exchanged during
3745 * LNI, is also be available at this point.
3746 */
3747 read_mgmt_allowed(dd, &ppd->mgmt_allowed);
3748 /* print the active widths */
3749 get_link_widths(dd, &active_tx, &active_rx);
3750 dd_dev_info(dd,
3751 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
3752 (int)power_management, (int)continious);
3753 dd_dev_info(dd,
3754 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
3755 (int)vau,
3756 (int)z,
3757 (int)vcu,
3758 (int)vl15buf,
3759 (int)partner_supported_crc);
3760 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
3761 (u32)remote_tx_rate, (u32)link_widths);
3762 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
3763 (u32)device_id, (u32)device_rev);
3764 /*
3765 * The peer vAU value just read is the peer receiver value. HFI does
3766 * not support a transmit vAU of 0 (AU == 8). We advertised that
3767 * with Z=1 in the fabric capabilities sent to the peer. The peer
3768 * will see our Z=1, and, if it advertised a vAU of 0, will move its
3769 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
3770 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
3771 * subject to the Z value exception.
3772 */
3773 if (vau == 0)
3774 vau = 1;
3775 set_up_vl15(dd, vau, vl15buf);
3776
3777 /* set up the LCB CRC mode */
3778 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
3779
3780 /* order is important: use the lowest bit in common */
3781 if (crc_mask & CAP_CRC_14B)
3782 crc_val = LCB_CRC_14B;
3783 else if (crc_mask & CAP_CRC_48B)
3784 crc_val = LCB_CRC_48B;
3785 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
3786 crc_val = LCB_CRC_12B_16B_PER_LANE;
3787 else
3788 crc_val = LCB_CRC_16B;
3789
3790 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
3791 write_csr(dd, DC_LCB_CFG_CRC_MODE,
3792 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
3793
3794 /* set (14b only) or clear sideband credit */
3795 reg = read_csr(dd, SEND_CM_CTRL);
3796 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
3797 write_csr(dd, SEND_CM_CTRL,
3798 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
3799 } else {
3800 write_csr(dd, SEND_CM_CTRL,
3801 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
3802 }
3803
3804 ppd->link_speed_active = 0; /* invalid value */
3805 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
3806 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
3807 switch (remote_tx_rate) {
3808 case 0:
3809 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
3810 break;
3811 case 1:
3812 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3813 break;
3814 }
3815 } else {
3816 /* actual rate is highest bit of the ANDed rates */
3817 u8 rate = remote_tx_rate & ppd->local_tx_rate;
3818
3819 if (rate & 2)
3820 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3821 else if (rate & 1)
3822 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
3823 }
3824 if (ppd->link_speed_active == 0) {
3825 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
3826 __func__, (int)remote_tx_rate);
3827 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3828 }
3829
3830 /*
3831 * Cache the values of the supported, enabled, and active
3832 * LTP CRC modes to return in 'portinfo' queries. But the bit
3833 * flags that are returned in the portinfo query differ from
3834 * what's in the link_crc_mask, crc_sizes, and crc_val
3835 * variables. Convert these here.
3836 */
3837 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
3838 /* supported crc modes */
3839 ppd->port_ltp_crc_mode |=
3840 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
3841 /* enabled crc modes */
3842 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
3843 /* active crc mode */
3844
3845 /* set up the remote credit return table */
3846 assign_remote_cm_au_table(dd, vcu);
3847
3848 /*
3849 * The LCB is reset on entry to handle_verify_cap(), so this must
3850 * be applied on every link up.
3851 *
3852 * Adjust LCB error kill enable to kill the link if
3853 * these RBUF errors are seen:
3854 * REPLAY_BUF_MBE_SMASK
3855 * FLIT_INPUT_BUF_MBE_SMASK
3856 */
995deafa 3857 if (is_ax(dd)) { /* fixed in B0 */
77241056
MM		 3858 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
3859 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
3860 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
3861 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
3862 }
3863
3864 /* pull LCB fifos out of reset - all fifo clocks must be stable */
3865 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
3866
3867 /* give 8051 access to the LCB CSRs */
3868 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
3869 set_8051_lcb_access(dd);
3870
3871 ppd->neighbor_guid =
3872 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
3873 ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
3874 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
3875 ppd->neighbor_type =
3876 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
3877 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
3878 ppd->neighbor_fm_security =
3879 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
3880 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
3881 dd_dev_info(dd,
3882 "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
3883 ppd->neighbor_guid, ppd->neighbor_type,
3884 ppd->mgmt_allowed, ppd->neighbor_fm_security);
3885 if (ppd->mgmt_allowed)
3886 add_full_mgmt_pkey(ppd);
3887
3888 /* tell the 8051 to go to LinkUp */
3889 set_link_state(ppd, HLS_GOING_UP);
3890}
3891
3892/*
3893 * Apply the link width downgrade enabled policy against the current active
3894 * link widths.
3895 *
3896 * Called when the enabled policy changes or the active link widths change.
3897 */
3898void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
3899{
77241056 3900 int do_bounce = 0;
323fd785
DL		 3901 int tries;
3902 u16 lwde;
77241056
MM		 3903 u16 tx, rx;
3904
323fd785
DL		 3905 /* use the hls lock to avoid a race with actual link up */
3906 tries = 0;
3907retry:
77241056
MM		 3908 mutex_lock(&ppd->hls_lock);
3909 /* only apply if the link is up */
323fd785
DL		 3910 if (!(ppd->host_link_state & HLS_UP)) {
3911 /* still going up..wait and retry */
3912 if (ppd->host_link_state & HLS_GOING_UP) {
3913 if (++tries < 1000) {
3914 mutex_unlock(&ppd->hls_lock);
3915 usleep_range(100, 120); /* arbitrary */
3916 goto retry;
3917 }
3918 dd_dev_err(ppd->dd,
3919 "%s: giving up waiting for link state change\n",
3920 __func__);
3921 }
3922 goto done;
3923 }
3924
3925 lwde = ppd->link_width_downgrade_enabled;
77241056
MM		 3926
3927 if (refresh_widths) {
3928 get_link_widths(ppd->dd, &tx, &rx);
3929 ppd->link_width_downgrade_tx_active = tx;
3930 ppd->link_width_downgrade_rx_active = rx;
3931 }
3932
3933 if (lwde == 0) {
3934 /* downgrade is disabled */
3935
3936 /* bounce if not at starting active width */
3937 if ((ppd->link_width_active !=
3938 ppd->link_width_downgrade_tx_active)
3939 || (ppd->link_width_active !=
3940 ppd->link_width_downgrade_rx_active)) {
3941 dd_dev_err(ppd->dd,
3942 "Link downgrade is disabled and link has downgraded, downing link\n");
3943 dd_dev_err(ppd->dd,
3944 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
3945 ppd->link_width_active,
3946 ppd->link_width_downgrade_tx_active,
3947 ppd->link_width_downgrade_rx_active);
3948 do_bounce = 1;
3949 }
3950 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0
3951 || (lwde & ppd->link_width_downgrade_rx_active) == 0) {
3952 /* Tx or Rx is outside the enabled policy */
3953 dd_dev_err(ppd->dd,
3954 "Link is outside of downgrade allowed, downing link\n");
3955 dd_dev_err(ppd->dd,
3956 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
3957 lwde,
3958 ppd->link_width_downgrade_tx_active,
3959 ppd->link_width_downgrade_rx_active);
3960 do_bounce = 1;
3961 }
3962
323fd785
DL		 3963done:
3964 mutex_unlock(&ppd->hls_lock);
3965
77241056
MM		 3966 if (do_bounce) {
3967 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
3968 OPA_LINKDOWN_REASON_WIDTH_POLICY);
3969 set_link_state(ppd, HLS_DN_OFFLINE);
3970 start_link(ppd);
3971 }
3972}
3973
3974/*
3975 * Handle a link downgrade interrupt from the 8051.
3976 *
3977 * This is a work-queue function outside of the interrupt.
3978 */
3979void handle_link_downgrade(struct work_struct *work)
3980{
3981 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3982 link_downgrade_work);
3983
3984 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
3985 apply_link_downgrade_policy(ppd, 1);
3986}
3987
3988static char *dcc_err_string(char *buf, int buf_len, u64 flags)
3989{
3990 return flag_string(buf, buf_len, flags, dcc_err_flags,
3991 ARRAY_SIZE(dcc_err_flags));
3992}
3993
3994static char *lcb_err_string(char *buf, int buf_len, u64 flags)
3995{
3996 return flag_string(buf, buf_len, flags, lcb_err_flags,
3997 ARRAY_SIZE(lcb_err_flags));
3998}
3999
4000static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
4001{
4002 return flag_string(buf, buf_len, flags, dc8051_err_flags,
4003 ARRAY_SIZE(dc8051_err_flags));
4004}
4005
4006static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
4007{
4008 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
4009 ARRAY_SIZE(dc8051_info_err_flags));
4010}
4011
4012static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
4013{
4014 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
4015 ARRAY_SIZE(dc8051_info_host_msg_flags));
4016}
4017
4018static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
4019{
4020 struct hfi1_pportdata *ppd = dd->pport;
4021 u64 info, err, host_msg;
4022 int queue_link_down = 0;
4023 char buf[96];
4024
4025 /* look at the flags */
4026 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
4027 /* 8051 information set by firmware */
4028 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
4029 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
4030 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
4031 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
4032 host_msg = (info >>
4033 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
4034 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
4035
4036 /*
4037 * Handle error flags.
4038 */
4039 if (err & FAILED_LNI) {
4040 /*
4041 * LNI error indications are cleared by the 8051
4042 * only when starting polling. Only pay attention
4043 * to them when in the states that occur during
4044 * LNI.
4045 */
4046 if (ppd->host_link_state
4047 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
4048 queue_link_down = 1;
4049 dd_dev_info(dd, "Link error: %s\n",
4050 dc8051_info_err_string(buf,
4051 sizeof(buf),
4052 err & FAILED_LNI));
4053 }
4054 err &= ~(u64)FAILED_LNI;
4055 }
4056 if (err) {
4057 /* report remaining errors, but do not do anything */
4058 dd_dev_err(dd, "8051 info error: %s\n",
4059 dc8051_info_err_string(buf, sizeof(buf), err));
4060 }
4061
4062 /*
4063 * Handle host message flags.
4064 */
4065 if (host_msg & HOST_REQ_DONE) {
4066 /*
4067 * Presently, the driver does a busy wait for
4068 * host requests to complete. This is only an
4069 * informational message.
4070 * NOTE: The 8051 clears the host message
4071 * information *on the next 8051 command*.
4072 * Therefore, when linkup is achieved,
4073 * this flag will still be set.
4074 */
4075 host_msg &= ~(u64)HOST_REQ_DONE;
4076 }
4077 if (host_msg & BC_SMA_MSG) {
4078 queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
4079 host_msg &= ~(u64)BC_SMA_MSG;
4080 }
4081 if (host_msg & LINKUP_ACHIEVED) {
4082 dd_dev_info(dd, "8051: Link up\n");
4083 queue_work(ppd->hfi1_wq, &ppd->link_up_work);
4084 host_msg &= ~(u64)LINKUP_ACHIEVED;
4085 }
4086 if (host_msg & EXT_DEVICE_CFG_REQ) {
4087 handle_8051_request(dd);
4088 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
4089 }
4090 if (host_msg & VERIFY_CAP_FRAME) {
4091 queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
4092 host_msg &= ~(u64)VERIFY_CAP_FRAME;
4093 }
4094 if (host_msg & LINK_GOING_DOWN) {
4095 const char *extra = "";
4096 /* no downgrade action needed if going down */
4097 if (host_msg & LINK_WIDTH_DOWNGRADED) {
4098 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
4099 extra = " (ignoring downgrade)";
4100 }
4101 dd_dev_info(dd, "8051: Link down%s\n", extra);
4102 queue_link_down = 1;
4103 host_msg &= ~(u64)LINK_GOING_DOWN;
4104 }
4105 if (host_msg & LINK_WIDTH_DOWNGRADED) {
4106 queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
4107 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
4108 }
4109 if (host_msg) {
4110 /* report remaining messages, but do not do anything */
4111 dd_dev_info(dd, "8051 info host message: %s\n",
4112 dc8051_info_host_msg_string(buf, sizeof(buf),
4113 host_msg));
4114 }
4115
4116 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
4117 }
4118 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
4119 /*
4120 * Lost the 8051 heartbeat. If this happens, we
4121 * receive constant interrupts about it. Disable
4122 * the interrupt after the first.
4123 */
4124 dd_dev_err(dd, "Lost 8051 heartbeat\n");
4125 write_csr(dd, DC_DC8051_ERR_EN,
4126 read_csr(dd, DC_DC8051_ERR_EN)
4127 & ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
4128
4129 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
4130 }
4131 if (reg) {
4132 /* report the error, but do not do anything */
4133 dd_dev_err(dd, "8051 error: %s\n",
4134 dc8051_err_string(buf, sizeof(buf), reg));
4135 }
4136
4137 if (queue_link_down) {
4138 /* if the link is already going down or disabled, do not
4139 * queue another */
4140 if ((ppd->host_link_state
4141 & (HLS_GOING_OFFLINE|HLS_LINK_COOLDOWN))
4142 || ppd->link_enabled == 0) {
4143 dd_dev_info(dd, "%s: not queuing link down\n",
4144 __func__);
4145 } else {
4146 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
4147 }
4148 }
4149}
4150
4151static const char * const fm_config_txt[] = {
4152[0] =
4153 "BadHeadDist: Distance violation between two head flits",
4154[1] =
4155 "BadTailDist: Distance violation between two tail flits",
4156[2] =
4157 "BadCtrlDist: Distance violation between two credit control flits",
4158[3] =
4159 "BadCrdAck: Credits return for unsupported VL",
4160[4] =
4161 "UnsupportedVLMarker: Received VL Marker",
4162[5] =
4163 "BadPreempt: Exceeded the preemption nesting level",
4164[6] =
4165 "BadControlFlit: Received unsupported control flit",
4166/* no 7 */
4167[8] =
4168 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
4169};
4170
4171static const char * const port_rcv_txt[] = {
4172[1] =
4173 "BadPktLen: Illegal PktLen",
4174[2] =
4175 "PktLenTooLong: Packet longer than PktLen",
4176[3] =
4177 "PktLenTooShort: Packet shorter than PktLen",
4178[4] =
4179 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
4180[5] =
4181 "BadDLID: Illegal DLID (0, doesn't match HFI)",
4182[6] =
4183 "BadL2: Illegal L2 opcode",
4184[7] =
4185 "BadSC: Unsupported SC",
4186[9] =
4187 "BadRC: Illegal RC",
4188[11] =
4189 "PreemptError: Preempting with same VL",
4190[12] =
4191 "PreemptVL15: Preempting a VL15 packet",
4192};
4193
4194#define OPA_LDR_FMCONFIG_OFFSET 16
4195#define OPA_LDR_PORTRCV_OFFSET 0
4196static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
4197{
4198 u64 info, hdr0, hdr1;
4199 const char *extra;
4200 char buf[96];
4201 struct hfi1_pportdata *ppd = dd->pport;
4202 u8 lcl_reason = 0;
4203 int do_bounce = 0;
4204
4205 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
4206 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
4207 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
4208 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
4209 /* set status bit */
4210 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
4211 }
4212 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
4213 }
4214
4215 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
4216 struct hfi1_pportdata *ppd = dd->pport;
4217 /* this counter saturates at (2^32) - 1 */
4218 if (ppd->link_downed < (u32)UINT_MAX)
4219 ppd->link_downed++;
4220 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
4221 }
4222
4223 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
4224 u8 reason_valid = 1;
4225
4226 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
4227 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
4228 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
4229 /* set status bit */
4230 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
4231 }
4232 switch (info) {
4233 case 0:
4234 case 1:
4235 case 2:
4236 case 3:
4237 case 4:
4238 case 5:
4239 case 6:
4240 extra = fm_config_txt[info];
4241 break;
4242 case 8:
4243 extra = fm_config_txt[info];
4244 if (ppd->port_error_action &
4245 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
4246 do_bounce = 1;
4247 /*
4248 * lcl_reason cannot be derived from info
4249 * for this error
4250 */
4251 lcl_reason =
4252 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
4253 }
4254 break;
4255 default:
4256 reason_valid = 0;
4257 snprintf(buf, sizeof(buf), "reserved%lld", info);
4258 extra = buf;
4259 break;
4260 }
4261
4262 if (reason_valid && !do_bounce) {
4263 do_bounce = ppd->port_error_action &
4264 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
4265 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
4266 }
4267
4268 /* just report this */
4269 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
4270 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
4271 }
4272
4273 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
4274 u8 reason_valid = 1;
4275
4276 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
4277 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
4278 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
4279 if (!(dd->err_info_rcvport.status_and_code &
4280 OPA_EI_STATUS_SMASK)) {
4281 dd->err_info_rcvport.status_and_code =
4282 info & OPA_EI_CODE_SMASK;
4283 /* set status bit */
4284 dd->err_info_rcvport.status_and_code |=
4285 OPA_EI_STATUS_SMASK;
4286 /* save first 2 flits in the packet that caused
4287 * the error */
4288 dd->err_info_rcvport.packet_flit1 = hdr0;
4289 dd->err_info_rcvport.packet_flit2 = hdr1;
4290 }
4291 switch (info) {
4292 case 1:
4293 case 2:
4294 case 3:
4295 case 4:
4296 case 5:
4297 case 6:
4298 case 7:
4299 case 9:
4300 case 11:
4301 case 12:
4302 extra = port_rcv_txt[info];
4303 break;
4304 default:
4305 reason_valid = 0;
4306 snprintf(buf, sizeof(buf), "reserved%lld", info);
4307 extra = buf;
4308 break;
4309 }
4310
4311 if (reason_valid && !do_bounce) {
4312 do_bounce = ppd->port_error_action &
4313 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
4314 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
4315 }
4316
4317 /* just report this */
4318 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
4319 dd_dev_info(dd, " hdr0 0x%llx, hdr1 0x%llx\n",
4320 hdr0, hdr1);
4321
4322 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
4323 }
4324
4325 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
4326 /* informative only */
4327 dd_dev_info(dd, "8051 access to LCB blocked\n");
4328 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
4329 }
4330 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
4331 /* informative only */
4332 dd_dev_info(dd, "host access to LCB blocked\n");
4333 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
4334 }
4335
4336 /* report any remaining errors */
4337 if (reg)
4338 dd_dev_info(dd, "DCC Error: %s\n",
4339 dcc_err_string(buf, sizeof(buf), reg));
4340
4341 if (lcl_reason == 0)
4342 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
4343
4344 if (do_bounce) {
4345 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
4346 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
4347 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
4348 }
4349}
4350
4351static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
4352{
4353 char buf[96];
4354
4355 dd_dev_info(dd, "LCB Error: %s\n",
4356 lcb_err_string(buf, sizeof(buf), reg));
4357}
4358
4359/*
4360 * CCE block DC interrupt. Source is < 8.
4361 */
4362static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
4363{
4364 const struct err_reg_info *eri = &dc_errs[source];
4365
4366 if (eri->handler) {
4367 interrupt_clear_down(dd, 0, eri);
4368 } else if (source == 3 /* dc_lbm_int */) {
4369 /*
4370 * This indicates that a parity error has occurred on the
4371 * address/control lines presented to the LBM. The error
4372 * is a single pulse, there is no associated error flag,
4373 * and it is non-maskable. This is because if a parity
4374 * error occurs on the request the request is dropped.
4375 * This should never occur, but it is nice to know if it
4376 * ever does.
4377 */
4378 dd_dev_err(dd, "Parity error in DC LBM block\n");
4379 } else {
4380 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
4381 }
4382}
4383
4384/*
4385 * TX block send credit interrupt. Source is < 160.
4386 */
4387static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
4388{
4389 sc_group_release_update(dd, source);
4390}
4391
4392/*
4393 * TX block SDMA interrupt. Source is < 48.
4394 *
4395 * SDMA interrupts are grouped by type:
4396 *
4397 * 0 - N-1 = SDma
4398 * N - 2N-1 = SDmaProgress
4399 * 2N - 3N-1 = SDmaIdle
4400 */
4401static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
4402{
4403 /* what interrupt */
4404 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
4405 /* which engine */
4406 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
4407
4408#ifdef CONFIG_SDMA_VERBOSITY
4409 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
4410 slashstrip(__FILE__), __LINE__, __func__);
4411 sdma_dumpstate(&dd->per_sdma[which]);
4412#endif
4413
4414 if (likely(what < 3 && which < dd->num_sdma)) {
4415 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
4416 } else {
4417 /* should not happen */
4418 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
4419 }
4420}
4421
4422/*
4423 * RX block receive available interrupt. Source is < 160.
4424 */
4425static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
4426{
4427 struct hfi1_ctxtdata *rcd;
4428 char *err_detail;
4429
4430 if (likely(source < dd->num_rcv_contexts)) {
4431 rcd = dd->rcd[source];
4432 if (rcd) {
4433 if (source < dd->first_user_ctxt)
f4f30031 4434 rcd->do_interrupt(rcd, 0);
77241056
MM		 4435 else
4436 handle_user_interrupt(rcd);
4437 return; /* OK */
4438 }
4439 /* received an interrupt, but no rcd */
4440 err_detail = "dataless";
4441 } else {
4442 /* received an interrupt, but are not using that context */
4443 err_detail = "out of range";
4444 }
4445 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
4446 err_detail, source);
4447}
4448
4449/*
4450 * RX block receive urgent interrupt. Source is < 160.
4451 */
4452static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
4453{
4454 struct hfi1_ctxtdata *rcd;
4455 char *err_detail;
4456
4457 if (likely(source < dd->num_rcv_contexts)) {
4458 rcd = dd->rcd[source];
4459 if (rcd) {
4460 /* only pay attention to user urgent interrupts */
4461 if (source >= dd->first_user_ctxt)
4462 handle_user_interrupt(rcd);
4463 return; /* OK */
4464 }
4465 /* received an interrupt, but no rcd */
4466 err_detail = "dataless";
4467 } else {
4468 /* received an interrupt, but are not using that context */
4469 err_detail = "out of range";
4470 }
4471 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
4472 err_detail, source);
4473}
4474
4475/*
4476 * Reserved range interrupt. Should not be called in normal operation.
4477 */
4478static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
4479{
4480 char name[64];
4481
4482 dd_dev_err(dd, "unexpected %s interrupt\n",
4483 is_reserved_name(name, sizeof(name), source));
4484}
4485
4486static const struct is_table is_table[] = {
4487/* start end
4488 name func interrupt func */
4489{ IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
4490 is_misc_err_name, is_misc_err_int },
4491{ IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
4492 is_sdma_eng_err_name, is_sdma_eng_err_int },
4493{ IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
4494 is_sendctxt_err_name, is_sendctxt_err_int },
4495{ IS_SDMA_START, IS_SDMA_END,
4496 is_sdma_eng_name, is_sdma_eng_int },
4497{ IS_VARIOUS_START, IS_VARIOUS_END,
4498 is_various_name, is_various_int },
4499{ IS_DC_START, IS_DC_END,
4500 is_dc_name, is_dc_int },
4501{ IS_RCVAVAIL_START, IS_RCVAVAIL_END,
4502 is_rcv_avail_name, is_rcv_avail_int },
4503{ IS_RCVURGENT_START, IS_RCVURGENT_END,
4504 is_rcv_urgent_name, is_rcv_urgent_int },
4505{ IS_SENDCREDIT_START, IS_SENDCREDIT_END,
4506 is_send_credit_name, is_send_credit_int},
4507{ IS_RESERVED_START, IS_RESERVED_END,
4508 is_reserved_name, is_reserved_int},
4509};
4510
4511/*
4512 * Interrupt source interrupt - called when the given source has an interrupt.
4513 * Source is a bit index into an array of 64-bit integers.
4514 */
4515static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
4516{
4517 const struct is_table *entry;
4518
4519 /* avoids a double compare by walking the table in-order */
4520 for (entry = &is_table[0]; entry->is_name; entry++) {
4521 if (source < entry->end) {
4522 trace_hfi1_interrupt(dd, entry, source);
4523 entry->is_int(dd, source - entry->start);
4524 return;
4525 }
4526 }
4527 /* fell off the end */
4528 dd_dev_err(dd, "invalid interrupt source %u\n", source);
4529}
4530
4531/*
4532 * General interrupt handler. This is able to correctly handle
4533 * all interrupts in case INTx is used.
4534 */
4535static irqreturn_t general_interrupt(int irq, void *data)
4536{
4537 struct hfi1_devdata *dd = data;
4538 u64 regs[CCE_NUM_INT_CSRS];
4539 u32 bit;
4540 int i;
4541
4542 this_cpu_inc(*dd->int_counter);
4543
4544 /* phase 1: scan and clear all handled interrupts */
4545 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
4546 if (dd->gi_mask[i] == 0) {
4547 regs[i] = 0; /* used later */
4548 continue;
4549 }
4550 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
4551 dd->gi_mask[i];
4552 /* only clear if anything is set */
4553 if (regs[i])
4554 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
4555 }
4556
4557 /* phase 2: call the appropriate handler */
4558 for_each_set_bit(bit, (unsigned long *)&regs[0],
4559 CCE_NUM_INT_CSRS*64) {
4560 is_interrupt(dd, bit);
4561 }
4562
4563 return IRQ_HANDLED;
4564}
4565
4566static irqreturn_t sdma_interrupt(int irq, void *data)
4567{
4568 struct sdma_engine *sde = data;
4569 struct hfi1_devdata *dd = sde->dd;
4570 u64 status;
4571
4572#ifdef CONFIG_SDMA_VERBOSITY
4573 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
4574 slashstrip(__FILE__), __LINE__, __func__);
4575 sdma_dumpstate(sde);
4576#endif
4577
4578 this_cpu_inc(*dd->int_counter);
4579
4580 /* This read_csr is really bad in the hot path */
4581 status = read_csr(dd,
4582 CCE_INT_STATUS + (8*(IS_SDMA_START/64)))
4583 & sde->imask;
4584 if (likely(status)) {
4585 /* clear the interrupt(s) */
4586 write_csr(dd,
4587 CCE_INT_CLEAR + (8*(IS_SDMA_START/64)),
4588 status);
4589
4590 /* handle the interrupt(s) */
4591 sdma_engine_interrupt(sde, status);
4592 } else
4593 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
4594 sde->this_idx);
4595
4596 return IRQ_HANDLED;
4597}
4598
4599/*
f4f30031
DL		 4600 * Clear the receive interrupt, forcing the write and making sure
4601 * we have data from the chip, pushing everything in front of it
4602 * back to the host.
4603 */
4604static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
4605{
4606 struct hfi1_devdata *dd = rcd->dd;
4607 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
4608
4609 mmiowb(); /* make sure everything before is written */
4610 write_csr(dd, addr, rcd->imask);
4611 /* force the above write on the chip and get a value back */
4612 (void)read_csr(dd, addr);
4613}
4614
4615/* force the receive interrupt */
4616static inline void force_recv_intr(struct hfi1_ctxtdata *rcd)
4617{
4618 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
4619}
4620
4621/* return non-zero if a packet is present */
4622static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
4623{
4624 if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
4625 return (rcd->seq_cnt ==
4626 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
4627
4628 /* else is RDMA rtail */
4629 return (rcd->head != get_rcvhdrtail(rcd));
4630}
4631
4632/*
4633 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
4634 * This routine will try to handle packets immediately (latency), but if
4635 * it finds too many, it will invoke the thread handler (bandwitdh). The
4636 * chip receive interupt is *not* cleared down until this or the thread (if
4637 * invoked) is finished. The intent is to avoid extra interrupts while we
4638 * are processing packets anyway.
77241056
MM		 4639 */
4640static irqreturn_t receive_context_interrupt(int irq, void *data)
4641{
4642 struct hfi1_ctxtdata *rcd = data;
4643 struct hfi1_devdata *dd = rcd->dd;
f4f30031
DL		 4644 int disposition;
4645 int present;
77241056
MM		 4646
4647 trace_hfi1_receive_interrupt(dd, rcd->ctxt);
4648 this_cpu_inc(*dd->int_counter);
4649
f4f30031
DL		 4650 /* receive interrupt remains blocked while processing packets */
4651 disposition = rcd->do_interrupt(rcd, 0);
77241056 4652
f4f30031
DL		 4653 /*
4654 * Too many packets were seen while processing packets in this
4655 * IRQ handler. Invoke the handler thread. The receive interrupt
4656 * remains blocked.
4657 */
4658 if (disposition == RCV_PKT_LIMIT)
4659 return IRQ_WAKE_THREAD;
4660
4661 /*
4662 * The packet processor detected no more packets. Clear the receive
4663 * interrupt and recheck for a packet packet that may have arrived
4664 * after the previous check and interrupt clear. If a packet arrived,
4665 * force another interrupt.
4666 */
4667 clear_recv_intr(rcd);
4668 present = check_packet_present(rcd);
4669 if (present)
4670 force_recv_intr(rcd);
4671
4672 return IRQ_HANDLED;
4673}
4674
4675/*
4676 * Receive packet thread handler. This expects to be invoked with the
4677 * receive interrupt still blocked.
4678 */
4679static irqreturn_t receive_context_thread(int irq, void *data)
4680{
4681 struct hfi1_ctxtdata *rcd = data;
4682 int present;
4683
4684 /* receive interrupt is still blocked from the IRQ handler */
4685 (void)rcd->do_interrupt(rcd, 1);
4686
4687 /*
4688 * The packet processor will only return if it detected no more
4689 * packets. Hold IRQs here so we can safely clear the interrupt and
4690 * recheck for a packet that may have arrived after the previous
4691 * check and the interrupt clear. If a packet arrived, force another
4692 * interrupt.
4693 */
4694 local_irq_disable();
4695 clear_recv_intr(rcd);
4696 present = check_packet_present(rcd);
4697 if (present)
4698 force_recv_intr(rcd);
4699 local_irq_enable();
77241056
MM		 4700
4701 return IRQ_HANDLED;
4702}
4703
4704/* ========================================================================= */
4705
4706u32 read_physical_state(struct hfi1_devdata *dd)
4707{
4708 u64 reg;
4709
4710 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
4711 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
4712 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
4713}
4714
4715static u32 read_logical_state(struct hfi1_devdata *dd)
4716{
4717 u64 reg;
4718
4719 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
4720 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
4721 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
4722}
4723
4724static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
4725{
4726 u64 reg;
4727
4728 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
4729 /* clear current state, set new state */
4730 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
4731 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
4732 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
4733}
4734
4735/*
4736 * Use the 8051 to read a LCB CSR.
4737 */
4738static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
4739{
4740 u32 regno;
4741 int ret;
4742
4743 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
4744 if (acquire_lcb_access(dd, 0) == 0) {
4745 *data = read_csr(dd, addr);
4746 release_lcb_access(dd, 0);
4747 return 0;
4748 }
4749 return -EBUSY;
4750 }
4751
4752 /* register is an index of LCB registers: (offset - base) / 8 */
4753 regno = (addr - DC_LCB_CFG_RUN) >> 3;
4754 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
4755 if (ret != HCMD_SUCCESS)
4756 return -EBUSY;
4757 return 0;
4758}
4759
4760/*
4761 * Read an LCB CSR. Access may not be in host control, so check.
4762 * Return 0 on success, -EBUSY on failure.
4763 */
4764int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
4765{
4766 struct hfi1_pportdata *ppd = dd->pport;
4767
4768 /* if up, go through the 8051 for the value */
4769 if (ppd->host_link_state & HLS_UP)
4770 return read_lcb_via_8051(dd, addr, data);
4771 /* if going up or down, no access */
4772 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
4773 return -EBUSY;
4774 /* otherwise, host has access */
4775 *data = read_csr(dd, addr);
4776 return 0;
4777}
4778
4779/*
4780 * Use the 8051 to write a LCB CSR.
4781 */
4782static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
4783{
3bf40d65
DL		 4784 u32 regno;
4785 int ret;
77241056 4786
3bf40d65
DL		 4787 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
4788 (dd->dc8051_ver < dc8051_ver(0, 20))) {
4789 if (acquire_lcb_access(dd, 0) == 0) {
4790 write_csr(dd, addr, data);
4791 release_lcb_access(dd, 0);
4792 return 0;
4793 }
4794 return -EBUSY;
77241056 4795 }
3bf40d65
DL		 4796
4797 /* register is an index of LCB registers: (offset - base) / 8 */
4798 regno = (addr - DC_LCB_CFG_RUN) >> 3;
4799 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
4800 if (ret != HCMD_SUCCESS)
4801 return -EBUSY;
4802 return 0;
77241056
MM		 4803}
4804
4805/*
4806 * Write an LCB CSR. Access may not be in host control, so check.
4807 * Return 0 on success, -EBUSY on failure.
4808 */
4809int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
4810{
4811 struct hfi1_pportdata *ppd = dd->pport;
4812
4813 /* if up, go through the 8051 for the value */
4814 if (ppd->host_link_state & HLS_UP)
4815 return write_lcb_via_8051(dd, addr, data);
4816 /* if going up or down, no access */
4817 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
4818 return -EBUSY;
4819 /* otherwise, host has access */
4820 write_csr(dd, addr, data);
4821 return 0;
4822}
4823
4824/*
4825 * Returns:
4826 * < 0 = Linux error, not able to get access
4827 * > 0 = 8051 command RETURN_CODE
4828 */
4829static int do_8051_command(
4830 struct hfi1_devdata *dd,
4831 u32 type,
4832 u64 in_data,
4833 u64 *out_data)
4834{
4835 u64 reg, completed;
4836 int return_code;
4837 unsigned long flags;
4838 unsigned long timeout;
4839
4840 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
4841
4842 /*
4843 * Alternative to holding the lock for a long time:
4844 * - keep busy wait - have other users bounce off
4845 */
4846 spin_lock_irqsave(&dd->dc8051_lock, flags);
4847
4848 /* We can't send any commands to the 8051 if it's in reset */
4849 if (dd->dc_shutdown) {
4850 return_code = -ENODEV;
4851 goto fail;
4852 }
4853
4854 /*
4855 * If an 8051 host command timed out previously, then the 8051 is
4856 * stuck.
4857 *
4858 * On first timeout, attempt to reset and restart the entire DC
4859 * block (including 8051). (Is this too big of a hammer?)
4860 *
4861 * If the 8051 times out a second time, the reset did not bring it
4862 * back to healthy life. In that case, fail any subsequent commands.
4863 */
4864 if (dd->dc8051_timed_out) {
4865 if (dd->dc8051_timed_out > 1) {
4866 dd_dev_err(dd,
4867 "Previous 8051 host command timed out, skipping command %u\n",
4868 type);
4869 return_code = -ENXIO;
4870 goto fail;
4871 }
4872 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
4873 dc_shutdown(dd);
4874 dc_start(dd);
4875 spin_lock_irqsave(&dd->dc8051_lock, flags);
4876 }
4877
4878 /*
4879 * If there is no timeout, then the 8051 command interface is
4880 * waiting for a command.
4881 */
4882
3bf40d65
DL		 4883 /*
4884 * When writing a LCB CSR, out_data contains the full value to
4885 * to be written, while in_data contains the relative LCB
4886 * address in 7:0. Do the work here, rather than the caller,
4887 * of distrubting the write data to where it needs to go:
4888 *
4889 * Write data
4890 * 39:00 -> in_data[47:8]
4891 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
4892 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
4893 */
4894 if (type == HCMD_WRITE_LCB_CSR) {
4895 in_data |= ((*out_data) & 0xffffffffffull) << 8;
4896 reg = ((((*out_data) >> 40) & 0xff) <<
4897 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
4898 | ((((*out_data) >> 48) & 0xffff) <<
4899 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
4900 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
4901 }
4902
77241056
MM		 4903 /*
4904 * Do two writes: the first to stabilize the type and req_data, the
4905 * second to activate.
4906 */
4907 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
4908 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
4909 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
4910 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
4911 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
4912 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
4913 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
4914
4915 /* wait for completion, alternate: interrupt */
4916 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
4917 while (1) {
4918 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
4919 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
4920 if (completed)
4921 break;
4922 if (time_after(jiffies, timeout)) {
4923 dd->dc8051_timed_out++;
4924 dd_dev_err(dd, "8051 host command %u timeout\n", type);
4925 if (out_data)
4926 *out_data = 0;
4927 return_code = -ETIMEDOUT;
4928 goto fail;
4929 }
4930 udelay(2);
4931 }
4932
4933 if (out_data) {
4934 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
4935 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
4936 if (type == HCMD_READ_LCB_CSR) {
4937 /* top 16 bits are in a different register */
4938 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
4939 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
4940 << (48
4941 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
4942 }
4943 }
4944 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
4945 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
4946 dd->dc8051_timed_out = 0;
4947 /*
4948 * Clear command for next user.
4949 */
4950 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
4951
4952fail:
4953 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
4954
4955 return return_code;
4956}
4957
4958static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
4959{
4960 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
4961}
4962
4963static int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
4964 u8 lane_id, u32 config_data)
4965{
4966 u64 data;
4967 int ret;
4968
4969 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
4970 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
4971 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
4972 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
4973 if (ret != HCMD_SUCCESS) {
4974 dd_dev_err(dd,
4975 "load 8051 config: field id %d, lane %d, err %d\n",
4976 (int)field_id, (int)lane_id, ret);
4977 }
4978 return ret;
4979}
4980
4981/*
4982 * Read the 8051 firmware "registers". Use the RAM directly. Always
4983 * set the result, even on error.
4984 * Return 0 on success, -errno on failure
4985 */
4986static int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
4987 u32 *result)
4988{
4989 u64 big_data;
4990 u32 addr;
4991 int ret;
4992
4993 /* address start depends on the lane_id */
4994 if (lane_id < 4)
4995 addr = (4 * NUM_GENERAL_FIELDS)
4996 + (lane_id * 4 * NUM_LANE_FIELDS);
4997 else
4998 addr = 0;
4999 addr += field_id * 4;
5000
5001 /* read is in 8-byte chunks, hardware will truncate the address down */
5002 ret = read_8051_data(dd, addr, 8, &big_data);
5003
5004 if (ret == 0) {
5005 /* extract the 4 bytes we want */
5006 if (addr & 0x4)
5007 *result = (u32)(big_data >> 32);
5008 else
5009 *result = (u32)big_data;
5010 } else {
5011 *result = 0;
5012 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
5013 __func__, lane_id, field_id);
5014 }
5015
5016 return ret;
5017}
5018
5019static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
5020 u8 continuous)
5021{
5022 u32 frame;
5023
5024 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
5025 | power_management << POWER_MANAGEMENT_SHIFT;
5026 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
5027 GENERAL_CONFIG, frame);
5028}
5029
5030static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
5031 u16 vl15buf, u8 crc_sizes)
5032{
5033 u32 frame;
5034
5035 frame = (u32)vau << VAU_SHIFT
5036 | (u32)z << Z_SHIFT
5037 | (u32)vcu << VCU_SHIFT
5038 | (u32)vl15buf << VL15BUF_SHIFT
5039 | (u32)crc_sizes << CRC_SIZES_SHIFT;
5040 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
5041 GENERAL_CONFIG, frame);
5042}
5043
5044static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
5045 u8 *flag_bits, u16 *link_widths)
5046{
5047 u32 frame;
5048
5049 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
5050 &frame);
5051 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
5052 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
5053 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
5054}
5055
5056static int write_vc_local_link_width(struct hfi1_devdata *dd,
5057 u8 misc_bits,
5058 u8 flag_bits,
5059 u16 link_widths)
5060{
5061 u32 frame;
5062
5063 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
5064 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
5065 | (u32)link_widths << LINK_WIDTH_SHIFT;
5066 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
5067 frame);
5068}
5069
5070static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
5071 u8 device_rev)
5072{
5073 u32 frame;
5074
5075 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
5076 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
5077 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
5078}
5079
5080static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
5081 u8 *device_rev)
5082{
5083 u32 frame;
5084
5085 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
5086 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
5087 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
5088 & REMOTE_DEVICE_REV_MASK;
5089}
5090
5091void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
5092{
5093 u32 frame;
5094
5095 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
5096 *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
5097 *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
5098}
5099
5100static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
5101 u8 *continuous)
5102{
5103 u32 frame;
5104
5105 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
5106 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
5107 & POWER_MANAGEMENT_MASK;
5108 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
5109 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
5110}
5111
5112static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
5113 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
5114{
5115 u32 frame;
5116
5117 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
5118 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
5119 *z = (frame >> Z_SHIFT) & Z_MASK;
5120 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
5121 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
5122 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
5123}
5124
5125static void read_vc_remote_link_width(struct hfi1_devdata *dd,
5126 u8 *remote_tx_rate,
5127 u16 *link_widths)
5128{
5129 u32 frame;
5130
5131 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
5132 &frame);
5133 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
5134 & REMOTE_TX_RATE_MASK;
5135 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
5136}
5137
5138static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
5139{
5140 u32 frame;
5141
5142 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
5143 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
5144}
5145
5146static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
5147{
5148 u32 frame;
5149
5150 read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
5151 *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
5152}
5153
5154static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
5155{
5156 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
5157}
5158
5159static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
5160{
5161 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
5162}
5163
5164void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
5165{
5166 u32 frame;
5167 int ret;
5168
5169 *link_quality = 0;
5170 if (dd->pport->host_link_state & HLS_UP) {
5171 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
5172 &frame);
5173 if (ret == 0)
5174 *link_quality = (frame >> LINK_QUALITY_SHIFT)
5175 & LINK_QUALITY_MASK;
5176 }
5177}
5178
5179static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
5180{
5181 u32 frame;
5182
5183 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
5184 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
5185}
5186
5187static int read_tx_settings(struct hfi1_devdata *dd,
5188 u8 *enable_lane_tx,
5189 u8 *tx_polarity_inversion,
5190 u8 *rx_polarity_inversion,
5191 u8 *max_rate)
5192{
5193 u32 frame;
5194 int ret;
5195
5196 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
5197 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
5198 & ENABLE_LANE_TX_MASK;
5199 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
5200 & TX_POLARITY_INVERSION_MASK;
5201 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
5202 & RX_POLARITY_INVERSION_MASK;
5203 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
5204 return ret;
5205}
5206
5207static int write_tx_settings(struct hfi1_devdata *dd,
5208 u8 enable_lane_tx,
5209 u8 tx_polarity_inversion,
5210 u8 rx_polarity_inversion,
5211 u8 max_rate)
5212{
5213 u32 frame;
5214
5215 /* no need to mask, all variable sizes match field widths */
5216 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
5217 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
5218 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
5219 | max_rate << MAX_RATE_SHIFT;
5220 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
5221}
5222
5223static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
5224{
5225 u32 frame, version, prod_id;
5226 int ret, lane;
5227
5228 /* 4 lanes */
5229 for (lane = 0; lane < 4; lane++) {
5230 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
5231 if (ret) {
5232 dd_dev_err(
5233 dd,
5234 "Unable to read lane %d firmware details\n",
5235 lane);
5236 continue;
5237 }
5238 version = (frame >> SPICO_ROM_VERSION_SHIFT)
5239 & SPICO_ROM_VERSION_MASK;
5240 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
5241 & SPICO_ROM_PROD_ID_MASK;
5242 dd_dev_info(dd,
5243 "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
5244 lane, version, prod_id);
5245 }
5246}
5247
5248/*
5249 * Read an idle LCB message.
5250 *
5251 * Returns 0 on success, -EINVAL on error
5252 */
5253static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
5254{
5255 int ret;
5256
5257 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG,
5258 type, data_out);
5259 if (ret != HCMD_SUCCESS) {
5260 dd_dev_err(dd, "read idle message: type %d, err %d\n",
5261 (u32)type, ret);
5262 return -EINVAL;
5263 }
5264 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
5265 /* return only the payload as we already know the type */
5266 *data_out >>= IDLE_PAYLOAD_SHIFT;
5267 return 0;
5268}
5269
5270/*
5271 * Read an idle SMA message. To be done in response to a notification from
5272 * the 8051.
5273 *
5274 * Returns 0 on success, -EINVAL on error
5275 */
5276static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
5277{
5278 return read_idle_message(dd,
5279 (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, data);
5280}
5281
5282/*
5283 * Send an idle LCB message.
5284 *
5285 * Returns 0 on success, -EINVAL on error
5286 */
5287static int send_idle_message(struct hfi1_devdata *dd, u64 data)
5288{
5289 int ret;
5290
5291 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
5292 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
5293 if (ret != HCMD_SUCCESS) {
5294 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
5295 data, ret);
5296 return -EINVAL;
5297 }
5298 return 0;
5299}
5300
5301/*
5302 * Send an idle SMA message.
5303 *
5304 * Returns 0 on success, -EINVAL on error
5305 */
5306int send_idle_sma(struct hfi1_devdata *dd, u64 message)
5307{
5308 u64 data;
5309
5310 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT)
5311 | ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
5312 return send_idle_message(dd, data);
5313}
5314
5315/*
5316 * Initialize the LCB then do a quick link up. This may or may not be
5317 * in loopback.
5318 *
5319 * return 0 on success, -errno on error
5320 */
5321static int do_quick_linkup(struct hfi1_devdata *dd)
5322{
5323 u64 reg;
5324 unsigned long timeout;
5325 int ret;
5326
5327 lcb_shutdown(dd, 0);
5328
5329 if (loopback) {
5330 /* LCB_CFG_LOOPBACK.VAL = 2 */
5331 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
5332 write_csr(dd, DC_LCB_CFG_LOOPBACK,
5333 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
5334 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
5335 }
5336
5337 /* start the LCBs */
5338 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
5339 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
5340
5341 /* simulator only loopback steps */
5342 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
5343 /* LCB_CFG_RUN.EN = 1 */
5344 write_csr(dd, DC_LCB_CFG_RUN,
5345 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
5346
5347 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
5348 timeout = jiffies + msecs_to_jiffies(10);
5349 while (1) {
5350 reg = read_csr(dd,
5351 DC_LCB_STS_LINK_TRANSFER_ACTIVE);
5352 if (reg)
5353 break;
5354 if (time_after(jiffies, timeout)) {
5355 dd_dev_err(dd,
5356 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
5357 return -ETIMEDOUT;
5358 }
5359 udelay(2);
5360 }
5361
5362 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
5363 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
5364 }
5365
5366 if (!loopback) {
5367 /*
5368 * When doing quick linkup and not in loopback, both
5369 * sides must be done with LCB set-up before either
5370 * starts the quick linkup. Put a delay here so that
5371 * both sides can be started and have a chance to be
5372 * done with LCB set up before resuming.
5373 */
5374 dd_dev_err(dd,
5375 "Pausing for peer to be finished with LCB set up\n");
5376 msleep(5000);
5377 dd_dev_err(dd,
5378 "Continuing with quick linkup\n");
5379 }
5380
5381 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
5382 set_8051_lcb_access(dd);
5383
5384 /*
5385 * State "quick" LinkUp request sets the physical link state to
5386 * LinkUp without a verify capability sequence.
5387 * This state is in simulator v37 and later.
5388 */
5389 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
5390 if (ret != HCMD_SUCCESS) {
5391 dd_dev_err(dd,
5392 "%s: set physical link state to quick LinkUp failed with return %d\n",
5393 __func__, ret);
5394
5395 set_host_lcb_access(dd);
5396 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
5397
5398 if (ret >= 0)
5399 ret = -EINVAL;
5400 return ret;
5401 }
5402
5403 return 0; /* success */
5404}
5405
5406/*
5407 * Set the SerDes to internal loopback mode.
5408 * Returns 0 on success, -errno on error.
5409 */
5410static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
5411{
5412 int ret;
5413
5414 ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
5415 if (ret == HCMD_SUCCESS)
5416 return 0;
5417 dd_dev_err(dd,
5418 "Set physical link state to SerDes Loopback failed with return %d\n",
5419 ret);
5420 if (ret >= 0)
5421 ret = -EINVAL;
5422 return ret;
5423}
5424
5425/*
5426 * Do all special steps to set up loopback.
5427 */
5428static int init_loopback(struct hfi1_devdata *dd)
5429{
5430 dd_dev_info(dd, "Entering loopback mode\n");
5431
5432 /* all loopbacks should disable self GUID check */
5433 write_csr(dd, DC_DC8051_CFG_MODE,
5434 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
5435
5436 /*
5437 * The simulator has only one loopback option - LCB. Switch
5438 * to that option, which includes quick link up.
5439 *
5440 * Accept all valid loopback values.
5441 */
5442 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
5443 && (loopback == LOOPBACK_SERDES
5444 || loopback == LOOPBACK_LCB
5445 || loopback == LOOPBACK_CABLE)) {
5446 loopback = LOOPBACK_LCB;
5447 quick_linkup = 1;
5448 return 0;
5449 }
5450
5451 /* handle serdes loopback */
5452 if (loopback == LOOPBACK_SERDES) {
5453 /* internal serdes loopack needs quick linkup on RTL */
5454 if (dd->icode == ICODE_RTL_SILICON)
5455 quick_linkup = 1;
5456 return set_serdes_loopback_mode(dd);
5457 }
5458
5459 /* LCB loopback - handled at poll time */
5460 if (loopback == LOOPBACK_LCB) {
5461 quick_linkup = 1; /* LCB is always quick linkup */
5462
5463 /* not supported in emulation due to emulation RTL changes */
5464 if (dd->icode == ICODE_FPGA_EMULATION) {
5465 dd_dev_err(dd,
5466 "LCB loopback not supported in emulation\n");
5467 return -EINVAL;
5468 }
5469 return 0;
5470 }
5471
5472 /* external cable loopback requires no extra steps */
5473 if (loopback == LOOPBACK_CABLE)
5474 return 0;
5475
5476 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
5477 return -EINVAL;
5478}
5479
5480/*
5481 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
5482 * used in the Verify Capability link width attribute.
5483 */
5484static u16 opa_to_vc_link_widths(u16 opa_widths)
5485{
5486 int i;
5487 u16 result = 0;
5488
5489 static const struct link_bits {
5490 u16 from;
5491 u16 to;
5492 } opa_link_xlate[] = {
5493 { OPA_LINK_WIDTH_1X, 1 << (1-1) },
5494 { OPA_LINK_WIDTH_2X, 1 << (2-1) },
5495 { OPA_LINK_WIDTH_3X, 1 << (3-1) },
5496 { OPA_LINK_WIDTH_4X, 1 << (4-1) },
5497 };
5498
5499 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
5500 if (opa_widths & opa_link_xlate[i].from)
5501 result |= opa_link_xlate[i].to;
5502 }
5503 return result;
5504}
5505
5506/*
5507 * Set link attributes before moving to polling.
5508 */
5509static int set_local_link_attributes(struct hfi1_pportdata *ppd)
5510{
5511 struct hfi1_devdata *dd = ppd->dd;
5512 u8 enable_lane_tx;
5513 u8 tx_polarity_inversion;
5514 u8 rx_polarity_inversion;
5515 int ret;
5516
5517 /* reset our fabric serdes to clear any lingering problems */
5518 fabric_serdes_reset(dd);
5519
5520 /* set the local tx rate - need to read-modify-write */
5521 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
5522 &rx_polarity_inversion, &ppd->local_tx_rate);
5523 if (ret)
5524 goto set_local_link_attributes_fail;
5525
5526 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
5527 /* set the tx rate to the fastest enabled */
5528 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
5529 ppd->local_tx_rate = 1;
5530 else
5531 ppd->local_tx_rate = 0;
5532 } else {
5533 /* set the tx rate to all enabled */
5534 ppd->local_tx_rate = 0;
5535 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
5536 ppd->local_tx_rate |= 2;
5537 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
5538 ppd->local_tx_rate |= 1;
5539 }
febffe2c
EH		 5540
5541 enable_lane_tx = 0xF; /* enable all four lanes */
77241056
MM		 5542 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
5543 rx_polarity_inversion, ppd->local_tx_rate);
5544 if (ret != HCMD_SUCCESS)
5545 goto set_local_link_attributes_fail;
5546
5547 /*
5548 * DC supports continuous updates.
5549 */
5550 ret = write_vc_local_phy(dd, 0 /* no power management */,
5551 1 /* continuous updates */);
5552 if (ret != HCMD_SUCCESS)
5553 goto set_local_link_attributes_fail;
5554
5555 /* z=1 in the next call: AU of 0 is not supported by the hardware */
5556 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
5557 ppd->port_crc_mode_enabled);
5558 if (ret != HCMD_SUCCESS)
5559 goto set_local_link_attributes_fail;
5560
5561 ret = write_vc_local_link_width(dd, 0, 0,
5562 opa_to_vc_link_widths(ppd->link_width_enabled));
5563 if (ret != HCMD_SUCCESS)
5564 goto set_local_link_attributes_fail;
5565
5566 /* let peer know who we are */
5567 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
5568 if (ret == HCMD_SUCCESS)
5569 return 0;
5570
5571set_local_link_attributes_fail:
5572 dd_dev_err(dd,
5573 "Failed to set local link attributes, return 0x%x\n",
5574 ret);
5575 return ret;
5576}
5577
5578/*
5579 * Call this to start the link. Schedule a retry if the cable is not
5580 * present or if unable to start polling. Do not do anything if the
5581 * link is disabled. Returns 0 if link is disabled or moved to polling
5582 */
5583int start_link(struct hfi1_pportdata *ppd)
5584{
5585 if (!ppd->link_enabled) {
5586 dd_dev_info(ppd->dd,
5587 "%s: stopping link start because link is disabled\n",
5588 __func__);
5589 return 0;
5590 }
5591 if (!ppd->driver_link_ready) {
5592 dd_dev_info(ppd->dd,
5593 "%s: stopping link start because driver is not ready\n",
5594 __func__);
5595 return 0;
5596 }
5597
5598 if (qsfp_mod_present(ppd) || loopback == LOOPBACK_SERDES ||
5599 loopback == LOOPBACK_LCB ||
5600 ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
5601 return set_link_state(ppd, HLS_DN_POLL);
5602
5603 dd_dev_info(ppd->dd,
5604 "%s: stopping link start because no cable is present\n",
5605 __func__);
5606 return -EAGAIN;
5607}
5608
5609static void reset_qsfp(struct hfi1_pportdata *ppd)
5610{
5611 struct hfi1_devdata *dd = ppd->dd;
5612 u64 mask, qsfp_mask;
5613
5614 mask = (u64)QSFP_HFI0_RESET_N;
5615 qsfp_mask = read_csr(dd,
5616 dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE);
5617 qsfp_mask |= mask;
5618 write_csr(dd,
5619 dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE,
5620 qsfp_mask);
5621
5622 qsfp_mask = read_csr(dd,
5623 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
5624 qsfp_mask &= ~mask;
5625 write_csr(dd,
5626 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT,
5627 qsfp_mask);
5628
5629 udelay(10);
5630
5631 qsfp_mask |= mask;
5632 write_csr(dd,
5633 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT,
5634 qsfp_mask);
5635}
5636
5637static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
5638 u8 *qsfp_interrupt_status)
5639{
5640 struct hfi1_devdata *dd = ppd->dd;
5641
5642 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
5643 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
5644 dd_dev_info(dd,
5645 "%s: QSFP cable on fire\n",
5646 __func__);
5647
5648 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
5649 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
5650 dd_dev_info(dd,
5651 "%s: QSFP cable temperature too low\n",
5652 __func__);
5653
5654 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
5655 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
5656 dd_dev_info(dd,
5657 "%s: QSFP supply voltage too high\n",
5658 __func__);
5659
5660 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
5661 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
5662 dd_dev_info(dd,
5663 "%s: QSFP supply voltage too low\n",
5664 __func__);
5665
5666 /* Byte 2 is vendor specific */
5667
5668 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
5669 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
5670 dd_dev_info(dd,
5671 "%s: Cable RX channel 1/2 power too high\n",
5672 __func__);
5673
5674 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
5675 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
5676 dd_dev_info(dd,
5677 "%s: Cable RX channel 1/2 power too low\n",
5678 __func__);
5679
5680 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
5681 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
5682 dd_dev_info(dd,
5683 "%s: Cable RX channel 3/4 power too high\n",
5684 __func__);
5685
5686 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
5687 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
5688 dd_dev_info(dd,
5689 "%s: Cable RX channel 3/4 power too low\n",
5690 __func__);
5691
5692 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
5693 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
5694 dd_dev_info(dd,
5695 "%s: Cable TX channel 1/2 bias too high\n",
5696 __func__);
5697
5698 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
5699 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
5700 dd_dev_info(dd,
5701 "%s: Cable TX channel 1/2 bias too low\n",
5702 __func__);
5703
5704 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
5705 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
5706 dd_dev_info(dd,
5707 "%s: Cable TX channel 3/4 bias too high\n",
5708 __func__);
5709
5710 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
5711 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
5712 dd_dev_info(dd,
5713 "%s: Cable TX channel 3/4 bias too low\n",
5714 __func__);
5715
5716 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
5717 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
5718 dd_dev_info(dd,
5719 "%s: Cable TX channel 1/2 power too high\n",
5720 __func__);
5721
5722 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
5723 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
5724 dd_dev_info(dd,
5725 "%s: Cable TX channel 1/2 power too low\n",
5726 __func__);
5727
5728 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
5729 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
5730 dd_dev_info(dd,
5731 "%s: Cable TX channel 3/4 power too high\n",
5732 __func__);
5733
5734 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
5735 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
5736 dd_dev_info(dd,
5737 "%s: Cable TX channel 3/4 power too low\n",
5738 __func__);
5739
5740 /* Bytes 9-10 and 11-12 are reserved */
5741 /* Bytes 13-15 are vendor specific */
5742
5743 return 0;
5744}
5745
5746static int do_pre_lni_host_behaviors(struct hfi1_pportdata *ppd)
5747{
5748 refresh_qsfp_cache(ppd, &ppd->qsfp_info);
5749
5750 return 0;
5751}
5752
5753static int do_qsfp_intr_fallback(struct hfi1_pportdata *ppd)
5754{
5755 struct hfi1_devdata *dd = ppd->dd;
5756 u8 qsfp_interrupt_status = 0;
5757
5758 if (qsfp_read(ppd, dd->hfi1_id, 2, &qsfp_interrupt_status, 1)
5759 != 1) {
5760 dd_dev_info(dd,
5761 "%s: Failed to read status of QSFP module\n",
5762 __func__);
5763 return -EIO;
5764 }
5765
5766 /* We don't care about alarms & warnings with a non-functional INT_N */
5767 if (!(qsfp_interrupt_status & QSFP_DATA_NOT_READY))
5768 do_pre_lni_host_behaviors(ppd);
5769
5770 return 0;
5771}
5772
5773/* This routine will only be scheduled if the QSFP module is present */
5774static void qsfp_event(struct work_struct *work)
5775{
5776 struct qsfp_data *qd;
5777 struct hfi1_pportdata *ppd;
5778 struct hfi1_devdata *dd;
5779
5780 qd = container_of(work, struct qsfp_data, qsfp_work);
5781 ppd = qd->ppd;
5782 dd = ppd->dd;
5783
5784 /* Sanity check */
5785 if (!qsfp_mod_present(ppd))
5786 return;
5787
5788 /*
5789 * Turn DC back on after cables has been
5790 * re-inserted. Up until now, the DC has been in
5791 * reset to save power.
5792 */
5793 dc_start(dd);
5794
5795 if (qd->cache_refresh_required) {
5796 msleep(3000);
5797 reset_qsfp(ppd);
5798
5799 /* Check for QSFP interrupt after t_init (SFF 8679)
5800 * + extra
5801 */
5802 msleep(3000);
5803 if (!qd->qsfp_interrupt_functional) {
5804 if (do_qsfp_intr_fallback(ppd) < 0)
5805 dd_dev_info(dd, "%s: QSFP fallback failed\n",
5806 __func__);
5807 ppd->driver_link_ready = 1;
5808 start_link(ppd);
5809 }
5810 }
5811
5812 if (qd->check_interrupt_flags) {
5813 u8 qsfp_interrupt_status[16] = {0,};
5814
5815 if (qsfp_read(ppd, dd->hfi1_id, 6,
5816 &qsfp_interrupt_status[0], 16) != 16) {
5817 dd_dev_info(dd,
5818 "%s: Failed to read status of QSFP module\n",
5819 __func__);
5820 } else {
5821 unsigned long flags;
5822 u8 data_status;
5823
5824 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5825 ppd->qsfp_info.check_interrupt_flags = 0;
5826 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
5827 flags);
5828
5829 if (qsfp_read(ppd, dd->hfi1_id, 2, &data_status, 1)
5830 != 1) {
5831 dd_dev_info(dd,
5832 "%s: Failed to read status of QSFP module\n",
5833 __func__);
5834 }
5835 if (!(data_status & QSFP_DATA_NOT_READY)) {
5836 do_pre_lni_host_behaviors(ppd);
5837 start_link(ppd);
5838 } else
5839 handle_qsfp_error_conditions(ppd,
5840 qsfp_interrupt_status);
5841 }
5842 }
5843}
5844
5845void init_qsfp(struct hfi1_pportdata *ppd)
5846{
5847 struct hfi1_devdata *dd = ppd->dd;
5848 u64 qsfp_mask;
5849
5850 if (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
3c2f85b8 5851 ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
77241056
MM		 5852 ppd->driver_link_ready = 1;
5853 return;
5854 }
5855
5856 ppd->qsfp_info.ppd = ppd;
5857 INIT_WORK(&ppd->qsfp_info.qsfp_work, qsfp_event);
5858
5859 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5860 /* Clear current status to avoid spurious interrupts */
5861 write_csr(dd,
5862 dd->hfi1_id ?
5863 ASIC_QSFP2_CLEAR :
5864 ASIC_QSFP1_CLEAR,
5865 qsfp_mask);
5866
5867 /* Handle active low nature of INT_N and MODPRST_N pins */
5868 if (qsfp_mod_present(ppd))
5869 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
5870 write_csr(dd,
5871 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
5872 qsfp_mask);
5873
5874 /* Allow only INT_N and MODPRST_N to trigger QSFP interrupts */
5875 qsfp_mask |= (u64)QSFP_HFI0_MODPRST_N;
5876 write_csr(dd,
5877 dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
5878 qsfp_mask);
5879
5880 if (qsfp_mod_present(ppd)) {
5881 msleep(3000);
5882 reset_qsfp(ppd);
5883
5884 /* Check for QSFP interrupt after t_init (SFF 8679)
5885 * + extra
5886 */
5887 msleep(3000);
5888 if (!ppd->qsfp_info.qsfp_interrupt_functional) {
5889 if (do_qsfp_intr_fallback(ppd) < 0)
5890 dd_dev_info(dd,
5891 "%s: QSFP fallback failed\n",
5892 __func__);
5893 ppd->driver_link_ready = 1;
5894 }
5895 }
5896}
5897
5898int bringup_serdes(struct hfi1_pportdata *ppd)
5899{
5900 struct hfi1_devdata *dd = ppd->dd;
5901 u64 guid;
5902 int ret;
5903
5904 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
5905 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
5906
5907 guid = ppd->guid;
5908 if (!guid) {
5909 if (dd->base_guid)
5910 guid = dd->base_guid + ppd->port - 1;
5911 ppd->guid = guid;
5912 }
5913
5914 /* the link defaults to enabled */
5915 ppd->link_enabled = 1;
5916 /* Set linkinit_reason on power up per OPA spec */
5917 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
5918
5919 if (loopback) {
5920 ret = init_loopback(dd);
5921 if (ret < 0)
5922 return ret;
5923 }
5924
5925 return start_link(ppd);
5926}
5927
5928void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
5929{
5930 struct hfi1_devdata *dd = ppd->dd;
5931
5932 /*
5933 * Shut down the link and keep it down. First turn off that the
5934 * driver wants to allow the link to be up (driver_link_ready).
5935 * Then make sure the link is not automatically restarted
5936 * (link_enabled). Cancel any pending restart. And finally
5937 * go offline.
5938 */
5939 ppd->driver_link_ready = 0;
5940 ppd->link_enabled = 0;
5941
5942 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
5943 OPA_LINKDOWN_REASON_SMA_DISABLED);
5944 set_link_state(ppd, HLS_DN_OFFLINE);
5945
5946 /* disable the port */
5947 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
5948}
5949
5950static inline int init_cpu_counters(struct hfi1_devdata *dd)
5951{
5952 struct hfi1_pportdata *ppd;
5953 int i;
5954
5955 ppd = (struct hfi1_pportdata *)(dd + 1);
5956 for (i = 0; i < dd->num_pports; i++, ppd++) {
5957 ppd->ibport_data.rc_acks = NULL;
5958 ppd->ibport_data.rc_qacks = NULL;
5959 ppd->ibport_data.rc_acks = alloc_percpu(u64);
5960 ppd->ibport_data.rc_qacks = alloc_percpu(u64);
5961 ppd->ibport_data.rc_delayed_comp = alloc_percpu(u64);
5962 if ((ppd->ibport_data.rc_acks == NULL) ||
5963 (ppd->ibport_data.rc_delayed_comp == NULL) ||
5964 (ppd->ibport_data.rc_qacks == NULL))
5965 return -ENOMEM;
5966 }
5967
5968 return 0;
5969}
5970
5971static const char * const pt_names[] = {
5972 "expected",
5973 "eager",
5974 "invalid"
5975};
5976
5977static const char *pt_name(u32 type)
5978{
5979 return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
5980}
5981
5982/*
5983 * index is the index into the receive array
5984 */
5985void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
5986 u32 type, unsigned long pa, u16 order)
5987{
5988 u64 reg;
5989 void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
5990 (dd->kregbase + RCV_ARRAY));
5991
5992 if (!(dd->flags & HFI1_PRESENT))
5993 goto done;
5994
5995 if (type == PT_INVALID) {
5996 pa = 0;
5997 } else if (type > PT_INVALID) {
5998 dd_dev_err(dd,
5999 "unexpected receive array type %u for index %u, not handled\n",
6000 type, index);
6001 goto done;
6002 }
6003
6004 hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
6005 pt_name(type), index, pa, (unsigned long)order);
6006
6007#define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
6008 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
6009 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
6010 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
6011 << RCV_ARRAY_RT_ADDR_SHIFT;
6012 writeq(reg, base + (index * 8));
6013
6014 if (type == PT_EAGER)
6015 /*
6016 * Eager entries are written one-by-one so we have to push them
6017 * after we write the entry.
6018 */
6019 flush_wc();
6020done:
6021 return;
6022}
6023
6024void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
6025{
6026 struct hfi1_devdata *dd = rcd->dd;
6027 u32 i;
6028
6029 /* this could be optimized */
6030 for (i = rcd->eager_base; i < rcd->eager_base +
6031 rcd->egrbufs.alloced; i++)
6032 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
6033
6034 for (i = rcd->expected_base;
6035 i < rcd->expected_base + rcd->expected_count; i++)
6036 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
6037}
6038
6039int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
6040 struct hfi1_ctxt_info *kinfo)
6041{
6042 kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
6043 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
6044 return 0;
6045}
6046
6047struct hfi1_message_header *hfi1_get_msgheader(
6048 struct hfi1_devdata *dd, __le32 *rhf_addr)
6049{
6050 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
6051
6052 return (struct hfi1_message_header *)
6053 (rhf_addr - dd->rhf_offset + offset);
6054}
6055
6056static const char * const ib_cfg_name_strings[] = {
6057 "HFI1_IB_CFG_LIDLMC",
6058 "HFI1_IB_CFG_LWID_DG_ENB",
6059 "HFI1_IB_CFG_LWID_ENB",
6060 "HFI1_IB_CFG_LWID",
6061 "HFI1_IB_CFG_SPD_ENB",
6062 "HFI1_IB_CFG_SPD",
6063 "HFI1_IB_CFG_RXPOL_ENB",
6064 "HFI1_IB_CFG_LREV_ENB",
6065 "HFI1_IB_CFG_LINKLATENCY",
6066 "HFI1_IB_CFG_HRTBT",
6067 "HFI1_IB_CFG_OP_VLS",
6068 "HFI1_IB_CFG_VL_HIGH_CAP",
6069 "HFI1_IB_CFG_VL_LOW_CAP",
6070 "HFI1_IB_CFG_OVERRUN_THRESH",
6071 "HFI1_IB_CFG_PHYERR_THRESH",
6072 "HFI1_IB_CFG_LINKDEFAULT",
6073 "HFI1_IB_CFG_PKEYS",
6074 "HFI1_IB_CFG_MTU",
6075 "HFI1_IB_CFG_LSTATE",
6076 "HFI1_IB_CFG_VL_HIGH_LIMIT",
6077 "HFI1_IB_CFG_PMA_TICKS",
6078 "HFI1_IB_CFG_PORT"
6079};
6080
6081static const char *ib_cfg_name(int which)
6082{
6083 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
6084 return "invalid";
6085 return ib_cfg_name_strings[which];
6086}
6087
6088int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
6089{
6090 struct hfi1_devdata *dd = ppd->dd;
6091 int val = 0;
6092
6093 switch (which) {
6094 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
6095 val = ppd->link_width_enabled;
6096 break;
6097 case HFI1_IB_CFG_LWID: /* currently active Link-width */
6098 val = ppd->link_width_active;
6099 break;
6100 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
6101 val = ppd->link_speed_enabled;
6102 break;
6103 case HFI1_IB_CFG_SPD: /* current Link speed */
6104 val = ppd->link_speed_active;
6105 break;
6106
6107 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
6108 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
6109 case HFI1_IB_CFG_LINKLATENCY:
6110 goto unimplemented;
6111
6112 case HFI1_IB_CFG_OP_VLS:
6113 val = ppd->vls_operational;
6114 break;
6115 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
6116 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
6117 break;
6118 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
6119 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
6120 break;
6121 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
6122 val = ppd->overrun_threshold;
6123 break;
6124 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
6125 val = ppd->phy_error_threshold;
6126 break;
6127 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
6128 val = dd->link_default;
6129 break;
6130
6131 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
6132 case HFI1_IB_CFG_PMA_TICKS:
6133 default:
6134unimplemented:
6135 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
6136 dd_dev_info(
6137 dd,
6138 "%s: which %s: not implemented\n",
6139 __func__,
6140 ib_cfg_name(which));
6141 break;
6142 }
6143
6144 return val;
6145}
6146
6147/*
6148 * The largest MAD packet size.
6149 */
6150#define MAX_MAD_PACKET 2048
6151
6152/*
6153 * Return the maximum header bytes that can go on the _wire_
6154 * for this device. This count includes the ICRC which is
6155 * not part of the packet held in memory but it is appended
6156 * by the HW.
6157 * This is dependent on the device's receive header entry size.
6158 * HFI allows this to be set per-receive context, but the
6159 * driver presently enforces a global value.
6160 */
6161u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
6162{
6163 /*
6164 * The maximum non-payload (MTU) bytes in LRH.PktLen are
6165 * the Receive Header Entry Size minus the PBC (or RHF) size
6166 * plus one DW for the ICRC appended by HW.
6167 *
6168 * dd->rcd[0].rcvhdrqentsize is in DW.
6169 * We use rcd[0] as all context will have the same value. Also,
6170 * the first kernel context would have been allocated by now so
6171 * we are guaranteed a valid value.
6172 */
6173 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
6174}
6175
6176/*
6177 * Set Send Length
6178 * @ppd - per port data
6179 *
6180 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
6181 * registers compare against LRH.PktLen, so use the max bytes included
6182 * in the LRH.
6183 *
6184 * This routine changes all VL values except VL15, which it maintains at
6185 * the same value.
6186 */
6187static void set_send_length(struct hfi1_pportdata *ppd)
6188{
6189 struct hfi1_devdata *dd = ppd->dd;
6190 u32 max_hb = lrh_max_header_bytes(dd), maxvlmtu = 0, dcmtu;
6191 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
6192 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
6193 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
6194 int i;
6195
6196 for (i = 0; i < ppd->vls_supported; i++) {
6197 if (dd->vld[i].mtu > maxvlmtu)
6198 maxvlmtu = dd->vld[i].mtu;
6199 if (i <= 3)
6200 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
6201 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
6202 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
6203 else
6204 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
6205 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
6206 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
6207 }
6208 write_csr(dd, SEND_LEN_CHECK0, len1);
6209 write_csr(dd, SEND_LEN_CHECK1, len2);
6210 /* adjust kernel credit return thresholds based on new MTUs */
6211 /* all kernel receive contexts have the same hdrqentsize */
6212 for (i = 0; i < ppd->vls_supported; i++) {
6213 sc_set_cr_threshold(dd->vld[i].sc,
6214 sc_mtu_to_threshold(dd->vld[i].sc, dd->vld[i].mtu,
6215 dd->rcd[0]->rcvhdrqentsize));
6216 }
6217 sc_set_cr_threshold(dd->vld[15].sc,
6218 sc_mtu_to_threshold(dd->vld[15].sc, dd->vld[15].mtu,
6219 dd->rcd[0]->rcvhdrqentsize));
6220
6221 /* Adjust maximum MTU for the port in DC */
6222 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
6223 (ilog2(maxvlmtu >> 8) + 1);
6224 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
6225 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
6226 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
6227 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
6228 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
6229}
6230
6231static void set_lidlmc(struct hfi1_pportdata *ppd)
6232{
6233 int i;
6234 u64 sreg = 0;
6235 struct hfi1_devdata *dd = ppd->dd;
6236 u32 mask = ~((1U << ppd->lmc) - 1);
6237 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
6238
6239 if (dd->hfi1_snoop.mode_flag)
6240 dd_dev_info(dd, "Set lid/lmc while snooping");
6241
6242 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
6243 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
6244 c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
6245 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT)|
6246 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
6247 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
6248 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
6249
6250 /*
6251 * Iterate over all the send contexts and set their SLID check
6252 */
6253 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
6254 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
6255 (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
6256 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
6257
6258 for (i = 0; i < dd->chip_send_contexts; i++) {
6259 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
6260 i, (u32)sreg);
6261 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
6262 }
6263
6264 /* Now we have to do the same thing for the sdma engines */
6265 sdma_update_lmc(dd, mask, ppd->lid);
6266}
6267
6268static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
6269{
6270 unsigned long timeout;
6271 u32 curr_state;
6272
6273 timeout = jiffies + msecs_to_jiffies(msecs);
6274 while (1) {
6275 curr_state = read_physical_state(dd);
6276 if (curr_state == state)
6277 break;
6278 if (time_after(jiffies, timeout)) {
6279 dd_dev_err(dd,
6280 "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
6281 state, curr_state);
6282 return -ETIMEDOUT;
6283 }
6284 usleep_range(1950, 2050); /* sleep 2ms-ish */
6285 }
6286
6287 return 0;
6288}
6289
6290/*
6291 * Helper for set_link_state(). Do not call except from that routine.
6292 * Expects ppd->hls_mutex to be held.
6293 *
6294 * @rem_reason value to be sent to the neighbor
6295 *
6296 * LinkDownReasons only set if transition succeeds.
6297 */
6298static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
6299{
6300 struct hfi1_devdata *dd = ppd->dd;
6301 u32 pstate, previous_state;
6302 u32 last_local_state;
6303 u32 last_remote_state;
6304 int ret;
6305 int do_transition;
6306 int do_wait;
6307
6308 previous_state = ppd->host_link_state;
6309 ppd->host_link_state = HLS_GOING_OFFLINE;
6310 pstate = read_physical_state(dd);
6311 if (pstate == PLS_OFFLINE) {
6312 do_transition = 0; /* in right state */
6313 do_wait = 0; /* ...no need to wait */
6314 } else if ((pstate & 0xff) == PLS_OFFLINE) {
6315 do_transition = 0; /* in an offline transient state */
6316 do_wait = 1; /* ...wait for it to settle */
6317 } else {
6318 do_transition = 1; /* need to move to offline */
6319 do_wait = 1; /* ...will need to wait */
6320 }
6321
6322 if (do_transition) {
6323 ret = set_physical_link_state(dd,
6324 PLS_OFFLINE | (rem_reason << 8));
6325
6326 if (ret != HCMD_SUCCESS) {
6327 dd_dev_err(dd,
6328 "Failed to transition to Offline link state, return %d\n",
6329 ret);
6330 return -EINVAL;
6331 }
6332 if (ppd->offline_disabled_reason == OPA_LINKDOWN_REASON_NONE)
6333 ppd->offline_disabled_reason =
6334 OPA_LINKDOWN_REASON_TRANSIENT;
6335 }
6336
6337 if (do_wait) {
6338 /* it can take a while for the link to go down */
dc060245 6339 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
77241056
MM		 6340 if (ret < 0)
6341 return ret;
6342 }
6343
6344 /* make sure the logical state is also down */
6345 wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
6346
6347 /*
6348 * Now in charge of LCB - must be after the physical state is
6349 * offline.quiet and before host_link_state is changed.
6350 */
6351 set_host_lcb_access(dd);
6352 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
6353 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
6354
6355 /*
6356 * The LNI has a mandatory wait time after the physical state
6357 * moves to Offline.Quiet. The wait time may be different
6358 * depending on how the link went down. The 8051 firmware
6359 * will observe the needed wait time and only move to ready
6360 * when that is completed. The largest of the quiet timeouts
6361 * is 2.5s, so wait that long and then a bit more.
6362 */
6363 ret = wait_fm_ready(dd, 3000);
6364 if (ret) {
6365 dd_dev_err(dd,
6366 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
6367 /* state is really offline, so make it so */
6368 ppd->host_link_state = HLS_DN_OFFLINE;
6369 return ret;
6370 }
6371
6372 /*
6373 * The state is now offline and the 8051 is ready to accept host
6374 * requests.
6375 * - change our state
6376 * - notify others if we were previously in a linkup state
6377 */
6378 ppd->host_link_state = HLS_DN_OFFLINE;
6379 if (previous_state & HLS_UP) {
6380 /* went down while link was up */
6381 handle_linkup_change(dd, 0);
6382 } else if (previous_state
6383 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
6384 /* went down while attempting link up */
6385 /* byte 1 of last_*_state is the failure reason */
6386 read_last_local_state(dd, &last_local_state);
6387 read_last_remote_state(dd, &last_remote_state);
6388 dd_dev_err(dd,
6389 "LNI failure last states: local 0x%08x, remote 0x%08x\n",
6390 last_local_state, last_remote_state);
6391 }
6392
6393 /* the active link width (downgrade) is 0 on link down */
6394 ppd->link_width_active = 0;
6395 ppd->link_width_downgrade_tx_active = 0;
6396 ppd->link_width_downgrade_rx_active = 0;
6397 ppd->current_egress_rate = 0;
6398 return 0;
6399}
6400
6401/* return the link state name */
6402static const char *link_state_name(u32 state)
6403{
6404 const char *name;
6405 int n = ilog2(state);
6406 static const char * const names[] = {
6407 [__HLS_UP_INIT_BP] = "INIT",
6408 [__HLS_UP_ARMED_BP] = "ARMED",
6409 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
6410 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
6411 [__HLS_DN_POLL_BP] = "POLL",
6412 [__HLS_DN_DISABLE_BP] = "DISABLE",
6413 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
6414 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
6415 [__HLS_GOING_UP_BP] = "GOING_UP",
6416 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
6417 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
6418 };
6419
6420 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
6421 return name ? name : "unknown";
6422}
6423
6424/* return the link state reason name */
6425static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
6426{
6427 if (state == HLS_UP_INIT) {
6428 switch (ppd->linkinit_reason) {
6429 case OPA_LINKINIT_REASON_LINKUP:
6430 return "(LINKUP)";
6431 case OPA_LINKINIT_REASON_FLAPPING:
6432 return "(FLAPPING)";
6433 case OPA_LINKINIT_OUTSIDE_POLICY:
6434 return "(OUTSIDE_POLICY)";
6435 case OPA_LINKINIT_QUARANTINED:
6436 return "(QUARANTINED)";
6437 case OPA_LINKINIT_INSUFIC_CAPABILITY:
6438 return "(INSUFIC_CAPABILITY)";
6439 default:
6440 break;
6441 }
6442 }
6443 return "";
6444}
6445
6446/*
6447 * driver_physical_state - convert the driver's notion of a port's
6448 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
6449 * Return -1 (converted to a u32) to indicate error.
6450 */
6451u32 driver_physical_state(struct hfi1_pportdata *ppd)
6452{
6453 switch (ppd->host_link_state) {
6454 case HLS_UP_INIT:
6455 case HLS_UP_ARMED:
6456 case HLS_UP_ACTIVE:
6457 return IB_PORTPHYSSTATE_LINKUP;
6458 case HLS_DN_POLL:
6459 return IB_PORTPHYSSTATE_POLLING;
6460 case HLS_DN_DISABLE:
6461 return IB_PORTPHYSSTATE_DISABLED;
6462 case HLS_DN_OFFLINE:
6463 return OPA_PORTPHYSSTATE_OFFLINE;
6464 case HLS_VERIFY_CAP:
6465 return IB_PORTPHYSSTATE_POLLING;
6466 case HLS_GOING_UP:
6467 return IB_PORTPHYSSTATE_POLLING;
6468 case HLS_GOING_OFFLINE:
6469 return OPA_PORTPHYSSTATE_OFFLINE;
6470 case HLS_LINK_COOLDOWN:
6471 return OPA_PORTPHYSSTATE_OFFLINE;
6472 case HLS_DN_DOWNDEF:
6473 default:
6474 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
6475 ppd->host_link_state);
6476 return -1;
6477 }
6478}
6479
6480/*
6481 * driver_logical_state - convert the driver's notion of a port's
6482 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
6483 * (converted to a u32) to indicate error.
6484 */
6485u32 driver_logical_state(struct hfi1_pportdata *ppd)
6486{
6487 if (ppd->host_link_state && !(ppd->host_link_state & HLS_UP))
6488 return IB_PORT_DOWN;
6489
6490 switch (ppd->host_link_state & HLS_UP) {
6491 case HLS_UP_INIT:
6492 return IB_PORT_INIT;
6493 case HLS_UP_ARMED:
6494 return IB_PORT_ARMED;
6495 case HLS_UP_ACTIVE:
6496 return IB_PORT_ACTIVE;
6497 default:
6498 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
6499 ppd->host_link_state);
6500 return -1;
6501 }
6502}
6503
6504void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
6505 u8 neigh_reason, u8 rem_reason)
6506{
6507 if (ppd->local_link_down_reason.latest == 0 &&
6508 ppd->neigh_link_down_reason.latest == 0) {
6509 ppd->local_link_down_reason.latest = lcl_reason;
6510 ppd->neigh_link_down_reason.latest = neigh_reason;
6511 ppd->remote_link_down_reason = rem_reason;
6512 }
6513}
6514
6515/*
6516 * Change the physical and/or logical link state.
6517 *
6518 * Do not call this routine while inside an interrupt. It contains
6519 * calls to routines that can take multiple seconds to finish.
6520 *
6521 * Returns 0 on success, -errno on failure.
6522 */
6523int set_link_state(struct hfi1_pportdata *ppd, u32 state)
6524{
6525 struct hfi1_devdata *dd = ppd->dd;
6526 struct ib_event event = {.device = NULL};
6527 int ret1, ret = 0;
6528 int was_up, is_down;
6529 int orig_new_state, poll_bounce;
6530
6531 mutex_lock(&ppd->hls_lock);
6532
6533 orig_new_state = state;
6534 if (state == HLS_DN_DOWNDEF)
6535 state = dd->link_default;
6536
6537 /* interpret poll -> poll as a link bounce */
6538 poll_bounce = ppd->host_link_state == HLS_DN_POLL
6539 && state == HLS_DN_POLL;
6540
6541 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
6542 link_state_name(ppd->host_link_state),
6543 link_state_name(orig_new_state),
6544 poll_bounce ? "(bounce) " : "",
6545 link_state_reason_name(ppd, state));
6546
6547 was_up = !!(ppd->host_link_state & HLS_UP);
6548
6549 /*
6550 * If we're going to a (HLS_*) link state that implies the logical
6551 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
6552 * reset is_sm_config_started to 0.
6553 */
6554 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
6555 ppd->is_sm_config_started = 0;
6556
6557 /*
6558 * Do nothing if the states match. Let a poll to poll link bounce
6559 * go through.
6560 */
6561 if (ppd->host_link_state == state && !poll_bounce)
6562 goto done;
6563
6564 switch (state) {
6565 case HLS_UP_INIT:
6566 if (ppd->host_link_state == HLS_DN_POLL && (quick_linkup
6567 || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
6568 /*
6569 * Quick link up jumps from polling to here.
6570 *
6571 * Whether in normal or loopback mode, the
6572 * simulator jumps from polling to link up.
6573 * Accept that here.
6574 */
6575 /* OK */;
6576 } else if (ppd->host_link_state != HLS_GOING_UP) {
6577 goto unexpected;
6578 }
6579
6580 ppd->host_link_state = HLS_UP_INIT;
6581 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
6582 if (ret) {
6583 /* logical state didn't change, stay at going_up */
6584 ppd->host_link_state = HLS_GOING_UP;
6585 dd_dev_err(dd,
6586 "%s: logical state did not change to INIT\n",
6587 __func__);
6588 } else {
6589 /* clear old transient LINKINIT_REASON code */
6590 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
6591 ppd->linkinit_reason =
6592 OPA_LINKINIT_REASON_LINKUP;
6593
6594 /* enable the port */
6595 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6596
6597 handle_linkup_change(dd, 1);
6598 }
6599 break;
6600 case HLS_UP_ARMED:
6601 if (ppd->host_link_state != HLS_UP_INIT)
6602 goto unexpected;
6603
6604 ppd->host_link_state = HLS_UP_ARMED;
6605 set_logical_state(dd, LSTATE_ARMED);
6606 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
6607 if (ret) {
6608 /* logical state didn't change, stay at init */
6609 ppd->host_link_state = HLS_UP_INIT;
6610 dd_dev_err(dd,
6611 "%s: logical state did not change to ARMED\n",
6612 __func__);
6613 }
6614 /*
6615 * The simulator does not currently implement SMA messages,
6616 * so neighbor_normal is not set. Set it here when we first
6617 * move to Armed.
6618 */
6619 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
6620 ppd->neighbor_normal = 1;
6621 break;
6622 case HLS_UP_ACTIVE:
6623 if (ppd->host_link_state != HLS_UP_ARMED)
6624 goto unexpected;
6625
6626 ppd->host_link_state = HLS_UP_ACTIVE;
6627 set_logical_state(dd, LSTATE_ACTIVE);
6628 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
6629 if (ret) {
6630 /* logical state didn't change, stay at armed */
6631 ppd->host_link_state = HLS_UP_ARMED;
6632 dd_dev_err(dd,
6633 "%s: logical state did not change to ACTIVE\n",
6634 __func__);
6635 } else {
6636
6637 /* tell all engines to go running */
6638 sdma_all_running(dd);
6639
6640 /* Signal the IB layer that the port has went active */
6641 event.device = &dd->verbs_dev.ibdev;
6642 event.element.port_num = ppd->port;
6643 event.event = IB_EVENT_PORT_ACTIVE;
6644 }
6645 break;
6646 case HLS_DN_POLL:
6647 if ((ppd->host_link_state == HLS_DN_DISABLE ||
6648 ppd->host_link_state == HLS_DN_OFFLINE) &&
6649 dd->dc_shutdown)
6650 dc_start(dd);
6651 /* Hand LED control to the DC */
6652 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
6653
6654 if (ppd->host_link_state != HLS_DN_OFFLINE) {
6655 u8 tmp = ppd->link_enabled;
6656
6657 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6658 if (ret) {
6659 ppd->link_enabled = tmp;
6660 break;
6661 }
6662 ppd->remote_link_down_reason = 0;
6663
6664 if (ppd->driver_link_ready)
6665 ppd->link_enabled = 1;
6666 }
6667
6668 ret = set_local_link_attributes(ppd);
6669 if (ret)
6670 break;
6671
6672 ppd->port_error_action = 0;
6673 ppd->host_link_state = HLS_DN_POLL;
6674
6675 if (quick_linkup) {
6676 /* quick linkup does not go into polling */
6677 ret = do_quick_linkup(dd);
6678 } else {
6679 ret1 = set_physical_link_state(dd, PLS_POLLING);
6680 if (ret1 != HCMD_SUCCESS) {
6681 dd_dev_err(dd,
6682 "Failed to transition to Polling link state, return 0x%x\n",
6683 ret1);
6684 ret = -EINVAL;
6685 }
6686 }
6687 ppd->offline_disabled_reason = OPA_LINKDOWN_REASON_NONE;
6688 /*
6689 * If an error occurred above, go back to offline. The
6690 * caller may reschedule another attempt.
6691 */
6692 if (ret)
6693 goto_offline(ppd, 0);
6694 break;
6695 case HLS_DN_DISABLE:
6696 /* link is disabled */
6697 ppd->link_enabled = 0;
6698
6699 /* allow any state to transition to disabled */
6700
6701 /* must transition to offline first */
6702 if (ppd->host_link_state != HLS_DN_OFFLINE) {
6703 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6704 if (ret)
6705 break;
6706 ppd->remote_link_down_reason = 0;
6707 }
6708
6709 ret1 = set_physical_link_state(dd, PLS_DISABLED);
6710 if (ret1 != HCMD_SUCCESS) {
6711 dd_dev_err(dd,
6712 "Failed to transition to Disabled link state, return 0x%x\n",
6713 ret1);
6714 ret = -EINVAL;
6715 break;
6716 }
6717 ppd->host_link_state = HLS_DN_DISABLE;
6718 dc_shutdown(dd);
6719 break;
6720 case HLS_DN_OFFLINE:
6721 if (ppd->host_link_state == HLS_DN_DISABLE)
6722 dc_start(dd);
6723
6724 /* allow any state to transition to offline */
6725 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6726 if (!ret)
6727 ppd->remote_link_down_reason = 0;
6728 break;
6729 case HLS_VERIFY_CAP:
6730 if (ppd->host_link_state != HLS_DN_POLL)
6731 goto unexpected;
6732 ppd->host_link_state = HLS_VERIFY_CAP;
6733 break;
6734 case HLS_GOING_UP:
6735 if (ppd->host_link_state != HLS_VERIFY_CAP)
6736 goto unexpected;
6737
6738 ret1 = set_physical_link_state(dd, PLS_LINKUP);
6739 if (ret1 != HCMD_SUCCESS) {
6740 dd_dev_err(dd,
6741 "Failed to transition to link up state, return 0x%x\n",
6742 ret1);
6743 ret = -EINVAL;
6744 break;
6745 }
6746 ppd->host_link_state = HLS_GOING_UP;
6747 break;
6748
6749 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
6750 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
6751 default:
6752 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
6753 __func__, state);
6754 ret = -EINVAL;
6755 break;
6756 }
6757
6758 is_down = !!(ppd->host_link_state & (HLS_DN_POLL |
6759 HLS_DN_DISABLE | HLS_DN_OFFLINE));
6760
6761 if (was_up && is_down && ppd->local_link_down_reason.sma == 0 &&
6762 ppd->neigh_link_down_reason.sma == 0) {
6763 ppd->local_link_down_reason.sma =
6764 ppd->local_link_down_reason.latest;
6765 ppd->neigh_link_down_reason.sma =
6766 ppd->neigh_link_down_reason.latest;
6767 }
6768
6769 goto done;
6770
6771unexpected:
6772 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
6773 __func__, link_state_name(ppd->host_link_state),
6774 link_state_name(state));
6775 ret = -EINVAL;
6776
6777done:
6778 mutex_unlock(&ppd->hls_lock);
6779
6780 if (event.device)
6781 ib_dispatch_event(&event);
6782
6783 return ret;
6784}
6785
6786int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
6787{
6788 u64 reg;
6789 int ret = 0;
6790
6791 switch (which) {
6792 case HFI1_IB_CFG_LIDLMC:
6793 set_lidlmc(ppd);
6794 break;
6795 case HFI1_IB_CFG_VL_HIGH_LIMIT:
6796 /*
6797 * The VL Arbitrator high limit is sent in units of 4k
6798 * bytes, while HFI stores it in units of 64 bytes.
6799 */
6800 val *= 4096/64;
6801 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
6802 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
6803 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
6804 break;
6805 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
6806 /* HFI only supports POLL as the default link down state */
6807 if (val != HLS_DN_POLL)
6808 ret = -EINVAL;
6809 break;
6810 case HFI1_IB_CFG_OP_VLS:
6811 if (ppd->vls_operational != val) {
6812 ppd->vls_operational = val;
6813 if (!ppd->port)
6814 ret = -EINVAL;
6815 else
6816 ret = sdma_map_init(
6817 ppd->dd,
6818 ppd->port - 1,
6819 val,
6820 NULL);
6821 }
6822 break;
6823 /*
6824 * For link width, link width downgrade, and speed enable, always AND
6825 * the setting with what is actually supported. This has two benefits.
6826 * First, enabled can't have unsupported values, no matter what the
6827 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
6828 * "fill in with your supported value" have all the bits in the
6829 * field set, so simply ANDing with supported has the desired result.
6830 */
6831 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
6832 ppd->link_width_enabled = val & ppd->link_width_supported;
6833 break;
6834 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
6835 ppd->link_width_downgrade_enabled =
6836 val & ppd->link_width_downgrade_supported;
6837 break;
6838 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
6839 ppd->link_speed_enabled = val & ppd->link_speed_supported;
6840 break;
6841 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
6842 /*
6843 * HFI does not follow IB specs, save this value
6844 * so we can report it, if asked.
6845 */
6846 ppd->overrun_threshold = val;
6847 break;
6848 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
6849 /*
6850 * HFI does not follow IB specs, save this value
6851 * so we can report it, if asked.
6852 */
6853 ppd->phy_error_threshold = val;
6854 break;
6855
6856 case HFI1_IB_CFG_MTU:
6857 set_send_length(ppd);
6858 break;
6859
6860 case HFI1_IB_CFG_PKEYS:
6861 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
6862 set_partition_keys(ppd);
6863 break;
6864
6865 default:
6866 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
6867 dd_dev_info(ppd->dd,
6868 "%s: which %s, val 0x%x: not implemented\n",
6869 __func__, ib_cfg_name(which), val);
6870 break;
6871 }
6872 return ret;
6873}
6874
6875/* begin functions related to vl arbitration table caching */
6876static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
6877{
6878 int i;
6879
6880 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
6881 VL_ARB_LOW_PRIO_TABLE_SIZE);
6882 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
6883 VL_ARB_HIGH_PRIO_TABLE_SIZE);
6884
6885 /*
6886 * Note that we always return values directly from the
6887 * 'vl_arb_cache' (and do no CSR reads) in response to a
6888 * 'Get(VLArbTable)'. This is obviously correct after a
6889 * 'Set(VLArbTable)', since the cache will then be up to
6890 * date. But it's also correct prior to any 'Set(VLArbTable)'
6891 * since then both the cache, and the relevant h/w registers
6892 * will be zeroed.
6893 */
6894
6895 for (i = 0; i < MAX_PRIO_TABLE; i++)
6896 spin_lock_init(&ppd->vl_arb_cache[i].lock);
6897}
6898
6899/*
6900 * vl_arb_lock_cache
6901 *
6902 * All other vl_arb_* functions should be called only after locking
6903 * the cache.
6904 */
6905static inline struct vl_arb_cache *
6906vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
6907{
6908 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
6909 return NULL;
6910 spin_lock(&ppd->vl_arb_cache[idx].lock);
6911 return &ppd->vl_arb_cache[idx];
6912}
6913
6914static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
6915{
6916 spin_unlock(&ppd->vl_arb_cache[idx].lock);
6917}
6918
6919static void vl_arb_get_cache(struct vl_arb_cache *cache,
6920 struct ib_vl_weight_elem *vl)
6921{
6922 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
6923}
6924
6925static void vl_arb_set_cache(struct vl_arb_cache *cache,
6926 struct ib_vl_weight_elem *vl)
6927{
6928 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
6929}
6930
6931static int vl_arb_match_cache(struct vl_arb_cache *cache,
6932 struct ib_vl_weight_elem *vl)
6933{
6934 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
6935}
6936/* end functions related to vl arbitration table caching */
6937
6938static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
6939 u32 size, struct ib_vl_weight_elem *vl)
6940{
6941 struct hfi1_devdata *dd = ppd->dd;
6942 u64 reg;
6943 unsigned int i, is_up = 0;
6944 int drain, ret = 0;
6945
6946 mutex_lock(&ppd->hls_lock);
6947
6948 if (ppd->host_link_state & HLS_UP)
6949 is_up = 1;
6950
6951 drain = !is_ax(dd) && is_up;
6952
6953 if (drain)
6954 /*
6955 * Before adjusting VL arbitration weights, empty per-VL
6956 * FIFOs, otherwise a packet whose VL weight is being
6957 * set to 0 could get stuck in a FIFO with no chance to
6958 * egress.
6959 */
6960 ret = stop_drain_data_vls(dd);
6961
6962 if (ret) {
6963 dd_dev_err(
6964 dd,
6965 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
6966 __func__);
6967 goto err;
6968 }
6969
6970 for (i = 0; i < size; i++, vl++) {
6971 /*
6972 * NOTE: The low priority shift and mask are used here, but
6973 * they are the same for both the low and high registers.
6974 */
6975 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
6976 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
6977 | (((u64)vl->weight
6978 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
6979 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
6980 write_csr(dd, target + (i * 8), reg);
6981 }
6982 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
6983
6984 if (drain)
6985 open_fill_data_vls(dd); /* reopen all VLs */
6986
6987err:
6988 mutex_unlock(&ppd->hls_lock);
6989
6990 return ret;
6991}
6992
6993/*
6994 * Read one credit merge VL register.
6995 */
6996static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
6997 struct vl_limit *vll)
6998{
6999 u64 reg = read_csr(dd, csr);
7000
7001 vll->dedicated = cpu_to_be16(
7002 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
7003 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
7004 vll->shared = cpu_to_be16(
7005 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
7006 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
7007}
7008
7009/*
7010 * Read the current credit merge limits.
7011 */
7012static int get_buffer_control(struct hfi1_devdata *dd,
7013 struct buffer_control *bc, u16 *overall_limit)
7014{
7015 u64 reg;
7016 int i;
7017
7018 /* not all entries are filled in */
7019 memset(bc, 0, sizeof(*bc));
7020
7021 /* OPA and HFI have a 1-1 mapping */
7022 for (i = 0; i < TXE_NUM_DATA_VL; i++)
7023 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8*i), &bc->vl[i]);
7024
7025 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
7026 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
7027
7028 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7029 bc->overall_shared_limit = cpu_to_be16(
7030 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
7031 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
7032 if (overall_limit)
7033 *overall_limit = (reg
7034 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
7035 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
7036 return sizeof(struct buffer_control);
7037}
7038
7039static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
7040{
7041 u64 reg;
7042 int i;
7043
7044 /* each register contains 16 SC->VLnt mappings, 4 bits each */
7045 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
7046 for (i = 0; i < sizeof(u64); i++) {
7047 u8 byte = *(((u8 *)&reg) + i);
7048
7049 dp->vlnt[2 * i] = byte & 0xf;
7050 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
7051 }
7052
7053 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
7054 for (i = 0; i < sizeof(u64); i++) {
7055 u8 byte = *(((u8 *)&reg) + i);
7056
7057 dp->vlnt[16 + (2 * i)] = byte & 0xf;
7058 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
7059 }
7060 return sizeof(struct sc2vlnt);
7061}
7062
7063static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
7064 struct ib_vl_weight_elem *vl)
7065{
7066 unsigned int i;
7067
7068 for (i = 0; i < nelems; i++, vl++) {
7069 vl->vl = 0xf;
7070 vl->weight = 0;
7071 }
7072}
7073
7074static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
7075{
7076 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
7077 DC_SC_VL_VAL(15_0,
7078 0, dp->vlnt[0] & 0xf,
7079 1, dp->vlnt[1] & 0xf,
7080 2, dp->vlnt[2] & 0xf,
7081 3, dp->vlnt[3] & 0xf,
7082 4, dp->vlnt[4] & 0xf,
7083 5, dp->vlnt[5] & 0xf,
7084 6, dp->vlnt[6] & 0xf,
7085 7, dp->vlnt[7] & 0xf,
7086 8, dp->vlnt[8] & 0xf,
7087 9, dp->vlnt[9] & 0xf,
7088 10, dp->vlnt[10] & 0xf,
7089 11, dp->vlnt[11] & 0xf,
7090 12, dp->vlnt[12] & 0xf,
7091 13, dp->vlnt[13] & 0xf,
7092 14, dp->vlnt[14] & 0xf,
7093 15, dp->vlnt[15] & 0xf));
7094 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
7095 DC_SC_VL_VAL(31_16,
7096 16, dp->vlnt[16] & 0xf,
7097 17, dp->vlnt[17] & 0xf,
7098 18, dp->vlnt[18] & 0xf,
7099 19, dp->vlnt[19] & 0xf,
7100 20, dp->vlnt[20] & 0xf,
7101 21, dp->vlnt[21] & 0xf,
7102 22, dp->vlnt[22] & 0xf,
7103 23, dp->vlnt[23] & 0xf,
7104 24, dp->vlnt[24] & 0xf,
7105 25, dp->vlnt[25] & 0xf,
7106 26, dp->vlnt[26] & 0xf,
7107 27, dp->vlnt[27] & 0xf,
7108 28, dp->vlnt[28] & 0xf,
7109 29, dp->vlnt[29] & 0xf,
7110 30, dp->vlnt[30] & 0xf,
7111 31, dp->vlnt[31] & 0xf));
7112}
7113
7114static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
7115 u16 limit)
7116{
7117 if (limit != 0)
7118 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
7119 what, (int)limit, idx);
7120}
7121
7122/* change only the shared limit portion of SendCmGLobalCredit */
7123static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
7124{
7125 u64 reg;
7126
7127 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7128 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
7129 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
7130 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
7131}
7132
7133/* change only the total credit limit portion of SendCmGLobalCredit */
7134static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
7135{
7136 u64 reg;
7137
7138 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7139 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
7140 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
7141 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
7142}
7143
7144/* set the given per-VL shared limit */
7145static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
7146{
7147 u64 reg;
7148 u32 addr;
7149
7150 if (vl < TXE_NUM_DATA_VL)
7151 addr = SEND_CM_CREDIT_VL + (8 * vl);
7152 else
7153 addr = SEND_CM_CREDIT_VL15;
7154
7155 reg = read_csr(dd, addr);
7156 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
7157 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
7158 write_csr(dd, addr, reg);
7159}
7160
7161/* set the given per-VL dedicated limit */
7162static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
7163{
7164 u64 reg;
7165 u32 addr;
7166
7167 if (vl < TXE_NUM_DATA_VL)
7168 addr = SEND_CM_CREDIT_VL + (8 * vl);
7169 else
7170 addr = SEND_CM_CREDIT_VL15;
7171
7172 reg = read_csr(dd, addr);
7173 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
7174 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
7175 write_csr(dd, addr, reg);
7176}
7177
7178/* spin until the given per-VL status mask bits clear */
7179static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
7180 const char *which)
7181{
7182 unsigned long timeout;
7183 u64 reg;
7184
7185 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
7186 while (1) {
7187 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
7188
7189 if (reg == 0)
7190 return; /* success */
7191 if (time_after(jiffies, timeout))
7192 break; /* timed out */
7193 udelay(1);
7194 }
7195
7196 dd_dev_err(dd,
7197 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
7198 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
7199 /*
7200 * If this occurs, it is likely there was a credit loss on the link.
7201 * The only recovery from that is a link bounce.
7202 */
7203 dd_dev_err(dd,
7204 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
7205}
7206
7207/*
7208 * The number of credits on the VLs may be changed while everything
7209 * is "live", but the following algorithm must be followed due to
7210 * how the hardware is actually implemented. In particular,
7211 * Return_Credit_Status[] is the only correct status check.
7212 *
7213 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
7214 * set Global_Shared_Credit_Limit = 0
7215 * use_all_vl = 1
7216 * mask0 = all VLs that are changing either dedicated or shared limits
7217 * set Shared_Limit[mask0] = 0
7218 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
7219 * if (changing any dedicated limit)
7220 * mask1 = all VLs that are lowering dedicated limits
7221 * lower Dedicated_Limit[mask1]
7222 * spin until Return_Credit_Status[mask1] == 0
7223 * raise Dedicated_Limits
7224 * raise Shared_Limits
7225 * raise Global_Shared_Credit_Limit
7226 *
7227 * lower = if the new limit is lower, set the limit to the new value
7228 * raise = if the new limit is higher than the current value (may be changed
7229 * earlier in the algorithm), set the new limit to the new value
7230 */
7231static int set_buffer_control(struct hfi1_devdata *dd,
7232 struct buffer_control *new_bc)
7233{
7234 u64 changing_mask, ld_mask, stat_mask;
7235 int change_count;
7236 int i, use_all_mask;
7237 int this_shared_changing;
7238 /*
7239 * A0: add the variable any_shared_limit_changing below and in the
7240 * algorithm above. If removing A0 support, it can be removed.
7241 */
7242 int any_shared_limit_changing;
7243 struct buffer_control cur_bc;
7244 u8 changing[OPA_MAX_VLS];
7245 u8 lowering_dedicated[OPA_MAX_VLS];
7246 u16 cur_total;
7247 u32 new_total = 0;
7248 const u64 all_mask =
7249 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
7250 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
7251 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
7252 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
7253 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
7254 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
7255 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
7256 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
7257 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
7258
7259#define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
7260#define NUM_USABLE_VLS 16 /* look at VL15 and less */
7261
7262
7263 /* find the new total credits, do sanity check on unused VLs */
7264 for (i = 0; i < OPA_MAX_VLS; i++) {
7265 if (valid_vl(i)) {
7266 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
7267 continue;
7268 }
7269 nonzero_msg(dd, i, "dedicated",
7270 be16_to_cpu(new_bc->vl[i].dedicated));
7271 nonzero_msg(dd, i, "shared",
7272 be16_to_cpu(new_bc->vl[i].shared));
7273 new_bc->vl[i].dedicated = 0;
7274 new_bc->vl[i].shared = 0;
7275 }
7276 new_total += be16_to_cpu(new_bc->overall_shared_limit);
7277 if (new_total > (u32)dd->link_credits)
7278 return -EINVAL;
7279 /* fetch the current values */
7280 get_buffer_control(dd, &cur_bc, &cur_total);
7281
7282 /*
7283 * Create the masks we will use.
7284 */
7285 memset(changing, 0, sizeof(changing));
7286 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
7287 /* NOTE: Assumes that the individual VL bits are adjacent and in
7288 increasing order */
7289 stat_mask =
7290 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
7291 changing_mask = 0;
7292 ld_mask = 0;
7293 change_count = 0;
7294 any_shared_limit_changing = 0;
7295 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
7296 if (!valid_vl(i))
7297 continue;
7298 this_shared_changing = new_bc->vl[i].shared
7299 != cur_bc.vl[i].shared;
7300 if (this_shared_changing)
7301 any_shared_limit_changing = 1;
7302 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated
7303 || this_shared_changing) {
7304 changing[i] = 1;
7305 changing_mask |= stat_mask;
7306 change_count++;
7307 }
7308 if (be16_to_cpu(new_bc->vl[i].dedicated) <
7309 be16_to_cpu(cur_bc.vl[i].dedicated)) {
7310 lowering_dedicated[i] = 1;
7311 ld_mask |= stat_mask;
7312 }
7313 }
7314
7315 /* bracket the credit change with a total adjustment */
7316 if (new_total > cur_total)
7317 set_global_limit(dd, new_total);
7318
7319 /*
7320 * Start the credit change algorithm.
7321 */
7322 use_all_mask = 0;
7323 if ((be16_to_cpu(new_bc->overall_shared_limit) <
995deafa
MM		 7324 be16_to_cpu(cur_bc.overall_shared_limit)) ||
7325 (is_ax(dd) && any_shared_limit_changing)) {
77241056
MM		 7326 set_global_shared(dd, 0);
7327 cur_bc.overall_shared_limit = 0;
7328 use_all_mask = 1;
7329 }
7330
7331 for (i = 0; i < NUM_USABLE_VLS; i++) {
7332 if (!valid_vl(i))
7333 continue;
7334
7335 if (changing[i]) {
7336 set_vl_shared(dd, i, 0);
7337 cur_bc.vl[i].shared = 0;
7338 }
7339 }
7340
7341 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
7342 "shared");
7343
7344 if (change_count > 0) {
7345 for (i = 0; i < NUM_USABLE_VLS; i++) {
7346 if (!valid_vl(i))
7347 continue;
7348
7349 if (lowering_dedicated[i]) {
7350 set_vl_dedicated(dd, i,
7351 be16_to_cpu(new_bc->vl[i].dedicated));
7352 cur_bc.vl[i].dedicated =
7353 new_bc->vl[i].dedicated;
7354 }
7355 }
7356
7357 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
7358
7359 /* now raise all dedicated that are going up */
7360 for (i = 0; i < NUM_USABLE_VLS; i++) {
7361 if (!valid_vl(i))
7362 continue;
7363
7364 if (be16_to_cpu(new_bc->vl[i].dedicated) >
7365 be16_to_cpu(cur_bc.vl[i].dedicated))
7366 set_vl_dedicated(dd, i,
7367 be16_to_cpu(new_bc->vl[i].dedicated));
7368 }
7369 }
7370
7371 /* next raise all shared that are going up */
7372 for (i = 0; i < NUM_USABLE_VLS; i++) {
7373 if (!valid_vl(i))
7374 continue;
7375
7376 if (be16_to_cpu(new_bc->vl[i].shared) >
7377 be16_to_cpu(cur_bc.vl[i].shared))
7378 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
7379 }
7380
7381 /* finally raise the global shared */
7382 if (be16_to_cpu(new_bc->overall_shared_limit) >
7383 be16_to_cpu(cur_bc.overall_shared_limit))
7384 set_global_shared(dd,
7385 be16_to_cpu(new_bc->overall_shared_limit));
7386
7387 /* bracket the credit change with a total adjustment */
7388 if (new_total < cur_total)
7389 set_global_limit(dd, new_total);
7390 return 0;
7391}
7392
7393/*
7394 * Read the given fabric manager table. Return the size of the
7395 * table (in bytes) on success, and a negative error code on
7396 * failure.
7397 */
7398int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
7399
7400{
7401 int size;
7402 struct vl_arb_cache *vlc;
7403
7404 switch (which) {
7405 case FM_TBL_VL_HIGH_ARB:
7406 size = 256;
7407 /*
7408 * OPA specifies 128 elements (of 2 bytes each), though
7409 * HFI supports only 16 elements in h/w.
7410 */
7411 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
7412 vl_arb_get_cache(vlc, t);
7413 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7414 break;
7415 case FM_TBL_VL_LOW_ARB:
7416 size = 256;
7417 /*
7418 * OPA specifies 128 elements (of 2 bytes each), though
7419 * HFI supports only 16 elements in h/w.
7420 */
7421 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
7422 vl_arb_get_cache(vlc, t);
7423 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7424 break;
7425 case FM_TBL_BUFFER_CONTROL:
7426 size = get_buffer_control(ppd->dd, t, NULL);
7427 break;
7428 case FM_TBL_SC2VLNT:
7429 size = get_sc2vlnt(ppd->dd, t);
7430 break;
7431 case FM_TBL_VL_PREEMPT_ELEMS:
7432 size = 256;
7433 /* OPA specifies 128 elements, of 2 bytes each */
7434 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
7435 break;
7436 case FM_TBL_VL_PREEMPT_MATRIX:
7437 size = 256;
7438 /*
7439 * OPA specifies that this is the same size as the VL
7440 * arbitration tables (i.e., 256 bytes).
7441 */
7442 break;
7443 default:
7444 return -EINVAL;
7445 }
7446 return size;
7447}
7448
7449/*
7450 * Write the given fabric manager table.
7451 */
7452int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
7453{
7454 int ret = 0;
7455 struct vl_arb_cache *vlc;
7456
7457 switch (which) {
7458 case FM_TBL_VL_HIGH_ARB:
7459 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
7460 if (vl_arb_match_cache(vlc, t)) {
7461 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7462 break;
7463 }
7464 vl_arb_set_cache(vlc, t);
7465 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7466 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
7467 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
7468 break;
7469 case FM_TBL_VL_LOW_ARB:
7470 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
7471 if (vl_arb_match_cache(vlc, t)) {
7472 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7473 break;
7474 }
7475 vl_arb_set_cache(vlc, t);
7476 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7477 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
7478 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
7479 break;
7480 case FM_TBL_BUFFER_CONTROL:
7481 ret = set_buffer_control(ppd->dd, t);
7482 break;
7483 case FM_TBL_SC2VLNT:
7484 set_sc2vlnt(ppd->dd, t);
7485 break;
7486 default:
7487 ret = -EINVAL;
7488 }
7489 return ret;
7490}
7491
7492/*
7493 * Disable all data VLs.
7494 *
7495 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
7496 */
7497static int disable_data_vls(struct hfi1_devdata *dd)
7498{
995deafa 7499 if (is_ax(dd))
77241056
MM		 7500 return 1;
7501
7502 pio_send_control(dd, PSC_DATA_VL_DISABLE);
7503
7504 return 0;
7505}
7506
7507/*
7508 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
7509 * Just re-enables all data VLs (the "fill" part happens
7510 * automatically - the name was chosen for symmetry with
7511 * stop_drain_data_vls()).
7512 *
7513 * Return 0 if successful, non-zero if the VLs cannot be enabled.
7514 */
7515int open_fill_data_vls(struct hfi1_devdata *dd)
7516{
995deafa 7517 if (is_ax(dd))
77241056
MM		 7518 return 1;
7519
7520 pio_send_control(dd, PSC_DATA_VL_ENABLE);
7521
7522 return 0;
7523}
7524
7525/*
7526 * drain_data_vls() - assumes that disable_data_vls() has been called,
7527 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
7528 * engines to drop to 0.
7529 */
7530static void drain_data_vls(struct hfi1_devdata *dd)
7531{
7532 sc_wait(dd);
7533 sdma_wait(dd);
7534 pause_for_credit_return(dd);
7535}
7536
7537/*
7538 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
7539 *
7540 * Use open_fill_data_vls() to resume using data VLs. This pair is
7541 * meant to be used like this:
7542 *
7543 * stop_drain_data_vls(dd);
7544 * // do things with per-VL resources
7545 * open_fill_data_vls(dd);
7546 */
7547int stop_drain_data_vls(struct hfi1_devdata *dd)
7548{
7549 int ret;
7550
7551 ret = disable_data_vls(dd);
7552 if (ret == 0)
7553 drain_data_vls(dd);
7554
7555 return ret;
7556}
7557
7558/*
7559 * Convert a nanosecond time to a cclock count. No matter how slow
7560 * the cclock, a non-zero ns will always have a non-zero result.
7561 */
7562u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
7563{
7564 u32 cclocks;
7565
7566 if (dd->icode == ICODE_FPGA_EMULATION)
7567 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
7568 else /* simulation pretends to be ASIC */
7569 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
7570 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
7571 cclocks = 1;
7572 return cclocks;
7573}
7574
7575/*
7576 * Convert a cclock count to nanoseconds. Not matter how slow
7577 * the cclock, a non-zero cclocks will always have a non-zero result.
7578 */
7579u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
7580{
7581 u32 ns;
7582
7583 if (dd->icode == ICODE_FPGA_EMULATION)
7584 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
7585 else /* simulation pretends to be ASIC */
7586 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
7587 if (cclocks && !ns)
7588 ns = 1;
7589 return ns;
7590}
7591
7592/*
7593 * Dynamically adjust the receive interrupt timeout for a context based on
7594 * incoming packet rate.
7595 *
7596 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
7597 */
7598static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
7599{
7600 struct hfi1_devdata *dd = rcd->dd;
7601 u32 timeout = rcd->rcvavail_timeout;
7602
7603 /*
7604 * This algorithm doubles or halves the timeout depending on whether
7605 * the number of packets received in this interrupt were less than or
7606 * greater equal the interrupt count.
7607 *
7608 * The calculations below do not allow a steady state to be achieved.
7609 * Only at the endpoints it is possible to have an unchanging
7610 * timeout.
7611 */
7612 if (npkts < rcv_intr_count) {
7613 /*
7614 * Not enough packets arrived before the timeout, adjust
7615 * timeout downward.
7616 */
7617 if (timeout < 2) /* already at minimum? */
7618 return;
7619 timeout >>= 1;
7620 } else {
7621 /*
7622 * More than enough packets arrived before the timeout, adjust
7623 * timeout upward.
7624 */
7625 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
7626 return;
7627 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
7628 }
7629
7630 rcd->rcvavail_timeout = timeout;
7631 /* timeout cannot be larger than rcv_intr_timeout_csr which has already
7632 been verified to be in range */
7633 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
7634 (u64)timeout << RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
7635}
7636
7637void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
7638 u32 intr_adjust, u32 npkts)
7639{
7640 struct hfi1_devdata *dd = rcd->dd;
7641 u64 reg;
7642 u32 ctxt = rcd->ctxt;
7643
7644 /*
7645 * Need to write timeout register before updating RcvHdrHead to ensure
7646 * that a new value is used when the HW decides to restart counting.
7647 */
7648 if (intr_adjust)
7649 adjust_rcv_timeout(rcd, npkts);
7650 if (updegr) {
7651 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
7652 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
7653 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
7654 }
7655 mmiowb();
7656 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
7657 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
7658 << RCV_HDR_HEAD_HEAD_SHIFT);
7659 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
7660 mmiowb();
7661}
7662
7663u32 hdrqempty(struct hfi1_ctxtdata *rcd)
7664{
7665 u32 head, tail;
7666
7667 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
7668 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
7669
7670 if (rcd->rcvhdrtail_kvaddr)
7671 tail = get_rcvhdrtail(rcd);
7672 else
7673 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
7674
7675 return head == tail;
7676}
7677
7678/*
7679 * Context Control and Receive Array encoding for buffer size:
7680 * 0x0 invalid
7681 * 0x1 4 KB
7682 * 0x2 8 KB
7683 * 0x3 16 KB
7684 * 0x4 32 KB
7685 * 0x5 64 KB
7686 * 0x6 128 KB
7687 * 0x7 256 KB
7688 * 0x8 512 KB (Receive Array only)
7689 * 0x9 1 MB (Receive Array only)
7690 * 0xa 2 MB (Receive Array only)
7691 *
7692 * 0xB-0xF - reserved (Receive Array only)
7693 *
7694 *
7695 * This routine assumes that the value has already been sanity checked.
7696 */
7697static u32 encoded_size(u32 size)
7698{
7699 switch (size) {
7700 case 4*1024: return 0x1;
7701 case 8*1024: return 0x2;
7702 case 16*1024: return 0x3;
7703 case 32*1024: return 0x4;
7704 case 64*1024: return 0x5;
7705 case 128*1024: return 0x6;
7706 case 256*1024: return 0x7;
7707 case 512*1024: return 0x8;
7708 case 1*1024*1024: return 0x9;
7709 case 2*1024*1024: return 0xa;
7710 }
7711 return 0x1; /* if invalid, go with the minimum size */
7712}
7713
7714void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
7715{
7716 struct hfi1_ctxtdata *rcd;
7717 u64 rcvctrl, reg;
7718 int did_enable = 0;
7719
7720 rcd = dd->rcd[ctxt];
7721 if (!rcd)
7722 return;
7723
7724 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
7725
7726 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
7727 /* if the context already enabled, don't do the extra steps */
7728 if ((op & HFI1_RCVCTRL_CTXT_ENB)
7729 && !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
7730 /* reset the tail and hdr addresses, and sequence count */
7731 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
7732 rcd->rcvhdrq_phys);
7733 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
7734 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7735 rcd->rcvhdrqtailaddr_phys);
7736 rcd->seq_cnt = 1;
7737
7738 /* reset the cached receive header queue head value */
7739 rcd->head = 0;
7740
7741 /*
7742 * Zero the receive header queue so we don't get false
7743 * positives when checking the sequence number. The
7744 * sequence numbers could land exactly on the same spot.
7745 * E.g. a rcd restart before the receive header wrapped.
7746 */
7747 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
7748
7749 /* starting timeout */
7750 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
7751
7752 /* enable the context */
7753 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
7754
7755 /* clean the egr buffer size first */
7756 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
7757 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
7758 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
7759 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
7760
7761 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
7762 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
7763 did_enable = 1;
7764
7765 /* zero RcvEgrIndexHead */
7766 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
7767
7768 /* set eager count and base index */
7769 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
7770 & RCV_EGR_CTRL_EGR_CNT_MASK)
7771 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
7772 (((rcd->eager_base >> RCV_SHIFT)
7773 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
7774 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
7775 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
7776
7777 /*
7778 * Set TID (expected) count and base index.
7779 * rcd->expected_count is set to individual RcvArray entries,
7780 * not pairs, and the CSR takes a pair-count in groups of
7781 * four, so divide by 8.
7782 */
7783 reg = (((rcd->expected_count >> RCV_SHIFT)
7784 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
7785 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
7786 (((rcd->expected_base >> RCV_SHIFT)
7787 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
7788 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
7789 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
82c2611d
NV		 7790 if (ctxt == HFI1_CTRL_CTXT)
7791 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
77241056
MM		 7792 }
7793 if (op & HFI1_RCVCTRL_CTXT_DIS) {
7794 write_csr(dd, RCV_VL15, 0);
46b010d3
MB		 7795 /*
7796 * When receive context is being disabled turn on tail
7797 * update with a dummy tail address and then disable
7798 * receive context.
7799 */
7800 if (dd->rcvhdrtail_dummy_physaddr) {
7801 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7802 dd->rcvhdrtail_dummy_physaddr);
7803 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7804 }
7805
77241056
MM		 7806 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
7807 }
7808 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
7809 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
7810 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
7811 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
7812 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
7813 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7814 if (op & HFI1_RCVCTRL_TAILUPD_DIS)
7815 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7816 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
7817 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
7818 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
7819 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
7820 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
7821 /* In one-packet-per-eager mode, the size comes from
7822 the RcvArray entry. */
7823 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
7824 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
7825 }
7826 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
7827 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
7828 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
7829 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
7830 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
7831 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
7832 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
7833 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
7834 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
7835 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
7836 rcd->rcvctrl = rcvctrl;
7837 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
7838 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
7839
7840 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
7841 if (did_enable
7842 && (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
7843 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
7844 if (reg != 0) {
7845 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
7846 ctxt, reg);
7847 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
7848 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
7849 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
7850 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
7851 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
7852 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
7853 ctxt, reg, reg == 0 ? "not" : "still");
7854 }
7855 }
7856
7857 if (did_enable) {
7858 /*
7859 * The interrupt timeout and count must be set after
7860 * the context is enabled to take effect.
7861 */
7862 /* set interrupt timeout */
7863 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
7864 (u64)rcd->rcvavail_timeout <<
7865 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
7866
7867 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
7868 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
7869 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
7870 }
7871
7872 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
7873 /*
7874 * If the context has been disabled and the Tail Update has
46b010d3
MB		 7875 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
7876 * so it doesn't contain an address that is invalid.
77241056 7877 */
46b010d3
MB		 7878 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7879 dd->rcvhdrtail_dummy_physaddr);
77241056
MM		 7880}
7881
7882u32 hfi1_read_cntrs(struct hfi1_devdata *dd, loff_t pos, char **namep,
7883 u64 **cntrp)
7884{
7885 int ret;
7886 u64 val = 0;
7887
7888 if (namep) {
7889 ret = dd->cntrnameslen;
7890 if (pos != 0) {
7891 dd_dev_err(dd, "read_cntrs does not support indexing");
7892 return 0;
7893 }
7894 *namep = dd->cntrnames;
7895 } else {
7896 const struct cntr_entry *entry;
7897 int i, j;
7898
7899 ret = (dd->ndevcntrs) * sizeof(u64);
7900 if (pos != 0) {
7901 dd_dev_err(dd, "read_cntrs does not support indexing");
7902 return 0;
7903 }
7904
7905 /* Get the start of the block of counters */
7906 *cntrp = dd->cntrs;
7907
7908 /*
7909 * Now go and fill in each counter in the block.
7910 */
7911 for (i = 0; i < DEV_CNTR_LAST; i++) {
7912 entry = &dev_cntrs[i];
7913 hfi1_cdbg(CNTR, "reading %s", entry->name);
7914 if (entry->flags & CNTR_DISABLED) {
7915 /* Nothing */
7916 hfi1_cdbg(CNTR, "\tDisabled\n");
7917 } else {
7918 if (entry->flags & CNTR_VL) {
7919 hfi1_cdbg(CNTR, "\tPer VL\n");
7920 for (j = 0; j < C_VL_COUNT; j++) {
7921 val = entry->rw_cntr(entry,
7922 dd, j,
7923 CNTR_MODE_R,
7924 0);
7925 hfi1_cdbg(
7926 CNTR,
7927 "\t\tRead 0x%llx for %d\n",
7928 val, j);
7929 dd->cntrs[entry->offset + j] =
7930 val;
7931 }
7932 } else {
7933 val = entry->rw_cntr(entry, dd,
7934 CNTR_INVALID_VL,
7935 CNTR_MODE_R, 0);
7936 dd->cntrs[entry->offset] = val;
7937 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
7938 }
7939 }
7940 }
7941 }
7942 return ret;
7943}
7944
7945/*
7946 * Used by sysfs to create files for hfi stats to read
7947 */
7948u32 hfi1_read_portcntrs(struct hfi1_devdata *dd, loff_t pos, u32 port,
7949 char **namep, u64 **cntrp)
7950{
7951 int ret;
7952 u64 val = 0;
7953
7954 if (namep) {
7955 ret = dd->portcntrnameslen;
7956 if (pos != 0) {
7957 dd_dev_err(dd, "index not supported");
7958 return 0;
7959 }
7960 *namep = dd->portcntrnames;
7961 } else {
7962 const struct cntr_entry *entry;
7963 struct hfi1_pportdata *ppd;
7964 int i, j;
7965
7966 ret = (dd->nportcntrs) * sizeof(u64);
7967 if (pos != 0) {
7968 dd_dev_err(dd, "indexing not supported");
7969 return 0;
7970 }
7971 ppd = (struct hfi1_pportdata *)(dd + 1 + port);
7972 *cntrp = ppd->cntrs;
7973
7974 for (i = 0; i < PORT_CNTR_LAST; i++) {
7975 entry = &port_cntrs[i];
7976 hfi1_cdbg(CNTR, "reading %s", entry->name);
7977 if (entry->flags & CNTR_DISABLED) {
7978 /* Nothing */
7979 hfi1_cdbg(CNTR, "\tDisabled\n");
7980 continue;
7981 }
7982
7983 if (entry->flags & CNTR_VL) {
7984 hfi1_cdbg(CNTR, "\tPer VL");
7985 for (j = 0; j < C_VL_COUNT; j++) {
7986 val = entry->rw_cntr(entry, ppd, j,
7987 CNTR_MODE_R,
7988 0);
7989 hfi1_cdbg(
7990 CNTR,
7991 "\t\tRead 0x%llx for %d",
7992 val, j);
7993 ppd->cntrs[entry->offset + j] = val;
7994 }
7995 } else {
7996 val = entry->rw_cntr(entry, ppd,
7997 CNTR_INVALID_VL,
7998 CNTR_MODE_R,
7999 0);
8000 ppd->cntrs[entry->offset] = val;
8001 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
8002 }
8003 }
8004 }
8005 return ret;
8006}
8007
8008static void free_cntrs(struct hfi1_devdata *dd)
8009{
8010 struct hfi1_pportdata *ppd;
8011 int i;
8012
8013 if (dd->synth_stats_timer.data)
8014 del_timer_sync(&dd->synth_stats_timer);
8015 dd->synth_stats_timer.data = 0;
8016 ppd = (struct hfi1_pportdata *)(dd + 1);
8017 for (i = 0; i < dd->num_pports; i++, ppd++) {
8018 kfree(ppd->cntrs);
8019 kfree(ppd->scntrs);
8020 free_percpu(ppd->ibport_data.rc_acks);
8021 free_percpu(ppd->ibport_data.rc_qacks);
8022 free_percpu(ppd->ibport_data.rc_delayed_comp);
8023 ppd->cntrs = NULL;
8024 ppd->scntrs = NULL;
8025 ppd->ibport_data.rc_acks = NULL;
8026 ppd->ibport_data.rc_qacks = NULL;
8027 ppd->ibport_data.rc_delayed_comp = NULL;
8028 }
8029 kfree(dd->portcntrnames);
8030 dd->portcntrnames = NULL;
8031 kfree(dd->cntrs);
8032 dd->cntrs = NULL;
8033 kfree(dd->scntrs);
8034 dd->scntrs = NULL;
8035 kfree(dd->cntrnames);
8036 dd->cntrnames = NULL;
8037}
8038
8039#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
8040#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
8041
8042static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
8043 u64 *psval, void *context, int vl)
8044{
8045 u64 val;
8046 u64 sval = *psval;
8047
8048 if (entry->flags & CNTR_DISABLED) {
8049 dd_dev_err(dd, "Counter %s not enabled", entry->name);
8050 return 0;
8051 }
8052
8053 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
8054
8055 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
8056
8057 /* If its a synthetic counter there is more work we need to do */
8058 if (entry->flags & CNTR_SYNTH) {
8059 if (sval == CNTR_MAX) {
8060 /* No need to read already saturated */
8061 return CNTR_MAX;
8062 }
8063
8064 if (entry->flags & CNTR_32BIT) {
8065 /* 32bit counters can wrap multiple times */
8066 u64 upper = sval >> 32;
8067 u64 lower = (sval << 32) >> 32;
8068
8069 if (lower > val) { /* hw wrapped */
8070 if (upper == CNTR_32BIT_MAX)
8071 val = CNTR_MAX;
8072 else
8073 upper++;
8074 }
8075
8076 if (val != CNTR_MAX)
8077 val = (upper << 32) | val;
8078
8079 } else {
8080 /* If we rolled we are saturated */
8081 if ((val < sval) || (val > CNTR_MAX))
8082 val = CNTR_MAX;
8083 }
8084 }
8085
8086 *psval = val;
8087
8088 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
8089
8090 return val;
8091}
8092
8093static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
8094 struct cntr_entry *entry,
8095 u64 *psval, void *context, int vl, u64 data)
8096{
8097 u64 val;
8098
8099 if (entry->flags & CNTR_DISABLED) {
8100 dd_dev_err(dd, "Counter %s not enabled", entry->name);
8101 return 0;
8102 }
8103
8104 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
8105
8106 if (entry->flags & CNTR_SYNTH) {
8107 *psval = data;
8108 if (entry->flags & CNTR_32BIT) {
8109 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
8110 (data << 32) >> 32);
8111 val = data; /* return the full 64bit value */
8112 } else {
8113 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
8114 data);
8115 }
8116 } else {
8117 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
8118 }
8119
8120 *psval = val;
8121
8122 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
8123
8124 return val;
8125}
8126
8127u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
8128{
8129 struct cntr_entry *entry;
8130 u64 *sval;
8131
8132 entry = &dev_cntrs[index];
8133 sval = dd->scntrs + entry->offset;
8134
8135 if (vl != CNTR_INVALID_VL)
8136 sval += vl;
8137
8138 return read_dev_port_cntr(dd, entry, sval, dd, vl);
8139}
8140
8141u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
8142{
8143 struct cntr_entry *entry;
8144 u64 *sval;
8145
8146 entry = &dev_cntrs[index];
8147 sval = dd->scntrs + entry->offset;
8148
8149 if (vl != CNTR_INVALID_VL)
8150 sval += vl;
8151
8152 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
8153}
8154
8155u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
8156{
8157 struct cntr_entry *entry;
8158 u64 *sval;
8159
8160 entry = &port_cntrs[index];
8161 sval = ppd->scntrs + entry->offset;
8162
8163 if (vl != CNTR_INVALID_VL)
8164 sval += vl;
8165
8166 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
8167 (index <= C_RCV_HDR_OVF_LAST)) {
8168 /* We do not want to bother for disabled contexts */
8169 return 0;
8170 }
8171
8172 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
8173}
8174
8175u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
8176{
8177 struct cntr_entry *entry;
8178 u64 *sval;
8179
8180 entry = &port_cntrs[index];
8181 sval = ppd->scntrs + entry->offset;
8182
8183 if (vl != CNTR_INVALID_VL)
8184 sval += vl;
8185
8186 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
8187 (index <= C_RCV_HDR_OVF_LAST)) {
8188 /* We do not want to bother for disabled contexts */
8189 return 0;
8190 }
8191
8192 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
8193}
8194
8195static void update_synth_timer(unsigned long opaque)
8196{
8197 u64 cur_tx;
8198 u64 cur_rx;
8199 u64 total_flits;
8200 u8 update = 0;
8201 int i, j, vl;
8202 struct hfi1_pportdata *ppd;
8203 struct cntr_entry *entry;
8204
8205 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
8206
8207 /*
8208 * Rather than keep beating on the CSRs pick a minimal set that we can
8209 * check to watch for potential roll over. We can do this by looking at
8210 * the number of flits sent/recv. If the total flits exceeds 32bits then
8211 * we have to iterate all the counters and update.
8212 */
8213 entry = &dev_cntrs[C_DC_RCV_FLITS];
8214 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
8215
8216 entry = &dev_cntrs[C_DC_XMIT_FLITS];
8217 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
8218
8219 hfi1_cdbg(
8220 CNTR,
8221 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
8222 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
8223
8224 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
8225 /*
8226 * May not be strictly necessary to update but it won't hurt and
8227 * simplifies the logic here.
8228 */
8229 update = 1;
8230 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
8231 dd->unit);
8232 } else {
8233 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
8234 hfi1_cdbg(CNTR,
8235 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
8236 total_flits, (u64)CNTR_32BIT_MAX);
8237 if (total_flits >= CNTR_32BIT_MAX) {
8238 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
8239 dd->unit);
8240 update = 1;
8241 }
8242 }
8243
8244 if (update) {
8245 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
8246 for (i = 0; i < DEV_CNTR_LAST; i++) {
8247 entry = &dev_cntrs[i];
8248 if (entry->flags & CNTR_VL) {
8249 for (vl = 0; vl < C_VL_COUNT; vl++)
8250 read_dev_cntr(dd, i, vl);
8251 } else {
8252 read_dev_cntr(dd, i, CNTR_INVALID_VL);
8253 }
8254 }
8255 ppd = (struct hfi1_pportdata *)(dd + 1);
8256 for (i = 0; i < dd->num_pports; i++, ppd++) {
8257 for (j = 0; j < PORT_CNTR_LAST; j++) {
8258 entry = &port_cntrs[j];
8259 if (entry->flags & CNTR_VL) {
8260 for (vl = 0; vl < C_VL_COUNT; vl++)
8261 read_port_cntr(ppd, j, vl);
8262 } else {
8263 read_port_cntr(ppd, j, CNTR_INVALID_VL);
8264 }
8265 }
8266 }
8267
8268 /*
8269 * We want the value in the register. The goal is to keep track
8270 * of the number of "ticks" not the counter value. In other
8271 * words if the register rolls we want to notice it and go ahead
8272 * and force an update.
8273 */
8274 entry = &dev_cntrs[C_DC_XMIT_FLITS];
8275 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
8276 CNTR_MODE_R, 0);
8277
8278 entry = &dev_cntrs[C_DC_RCV_FLITS];
8279 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
8280 CNTR_MODE_R, 0);
8281
8282 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
8283 dd->unit, dd->last_tx, dd->last_rx);
8284
8285 } else {
8286 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
8287 }
8288
8289mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
8290}
8291
8292#define C_MAX_NAME 13 /* 12 chars + one for /0 */
8293static int init_cntrs(struct hfi1_devdata *dd)
8294{
8295 int i, rcv_ctxts, index, j;
8296 size_t sz;
8297 char *p;
8298 char name[C_MAX_NAME];
8299 struct hfi1_pportdata *ppd;
8300
8301 /* set up the stats timer; the add_timer is done at the end */
24523a94
MFW		 8302 setup_timer(&dd->synth_stats_timer, update_synth_timer,
8303 (unsigned long)dd);
77241056
MM		 8304
8305 /***********************/
8306 /* per device counters */
8307 /***********************/
8308
8309 /* size names and determine how many we have*/
8310 dd->ndevcntrs = 0;
8311 sz = 0;
8312 index = 0;
8313
8314 for (i = 0; i < DEV_CNTR_LAST; i++) {
8315 hfi1_dbg_early("Init cntr %s\n", dev_cntrs[i].name);
8316 if (dev_cntrs[i].flags & CNTR_DISABLED) {
8317 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
8318 continue;
8319 }
8320
8321 if (dev_cntrs[i].flags & CNTR_VL) {
8322 hfi1_dbg_early("\tProcessing VL cntr\n");
8323 dev_cntrs[i].offset = index;
8324 for (j = 0; j < C_VL_COUNT; j++) {
8325 memset(name, '\0', C_MAX_NAME);
8326 snprintf(name, C_MAX_NAME, "%s%d",
8327 dev_cntrs[i].name,
8328 vl_from_idx(j));
8329 sz += strlen(name);
8330 sz++;
8331 hfi1_dbg_early("\t\t%s\n", name);
8332 dd->ndevcntrs++;
8333 index++;
8334 }
8335 } else {
8336 /* +1 for newline */
8337 sz += strlen(dev_cntrs[i].name) + 1;
8338 dd->ndevcntrs++;
8339 dev_cntrs[i].offset = index;
8340 index++;
8341 hfi1_dbg_early("\tAdding %s\n", dev_cntrs[i].name);
8342 }
8343 }
8344
8345 /* allocate space for the counter values */
8346 dd->cntrs = kcalloc(index, sizeof(u64), GFP_KERNEL);
8347 if (!dd->cntrs)
8348 goto bail;
8349
8350 dd->scntrs = kcalloc(index, sizeof(u64), GFP_KERNEL);
8351 if (!dd->scntrs)
8352 goto bail;
8353
8354
8355 /* allocate space for the counter names */
8356 dd->cntrnameslen = sz;
8357 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
8358 if (!dd->cntrnames)
8359 goto bail;
8360
8361 /* fill in the names */
8362 for (p = dd->cntrnames, i = 0, index = 0; i < DEV_CNTR_LAST; i++) {
8363 if (dev_cntrs[i].flags & CNTR_DISABLED) {
8364 /* Nothing */
8365 } else {
8366 if (dev_cntrs[i].flags & CNTR_VL) {
8367 for (j = 0; j < C_VL_COUNT; j++) {
8368 memset(name, '\0', C_MAX_NAME);
8369 snprintf(name, C_MAX_NAME, "%s%d",
8370 dev_cntrs[i].name,
8371 vl_from_idx(j));
8372 memcpy(p, name, strlen(name));
8373 p += strlen(name);
8374 *p++ = '\n';
8375 }
8376 } else {
8377 memcpy(p, dev_cntrs[i].name,
8378 strlen(dev_cntrs[i].name));
8379 p += strlen(dev_cntrs[i].name);
8380 *p++ = '\n';
8381 }
8382 index++;
8383 }
8384 }
8385
8386 /*********************/
8387 /* per port counters */
8388 /*********************/
8389
8390 /*
8391 * Go through the counters for the overflows and disable the ones we
8392 * don't need. This varies based on platform so we need to do it
8393 * dynamically here.
8394 */
8395 rcv_ctxts = dd->num_rcv_contexts;
8396 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
8397 i <= C_RCV_HDR_OVF_LAST; i++) {
8398 port_cntrs[i].flags |= CNTR_DISABLED;
8399 }
8400
8401 /* size port counter names and determine how many we have*/
8402 sz = 0;
8403 dd->nportcntrs = 0;
8404 for (i = 0; i < PORT_CNTR_LAST; i++) {
8405 hfi1_dbg_early("Init pcntr %s\n", port_cntrs[i].name);
8406 if (port_cntrs[i].flags & CNTR_DISABLED) {
8407 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
8408 continue;
8409 }
8410
8411 if (port_cntrs[i].flags & CNTR_VL) {
8412 hfi1_dbg_early("\tProcessing VL cntr\n");
8413 port_cntrs[i].offset = dd->nportcntrs;
8414 for (j = 0; j < C_VL_COUNT; j++) {
8415 memset(name, '\0', C_MAX_NAME);
8416 snprintf(name, C_MAX_NAME, "%s%d",
8417 port_cntrs[i].name,
8418 vl_from_idx(j));
8419 sz += strlen(name);
8420 sz++;
8421 hfi1_dbg_early("\t\t%s\n", name);
8422 dd->nportcntrs++;
8423 }
8424 } else {
8425 /* +1 for newline */
8426 sz += strlen(port_cntrs[i].name) + 1;
8427 port_cntrs[i].offset = dd->nportcntrs;
8428 dd->nportcntrs++;
8429 hfi1_dbg_early("\tAdding %s\n", port_cntrs[i].name);
8430 }
8431 }
8432
8433 /* allocate space for the counter names */
8434 dd->portcntrnameslen = sz;
8435 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
8436 if (!dd->portcntrnames)
8437 goto bail;
8438
8439 /* fill in port cntr names */
8440 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
8441 if (port_cntrs[i].flags & CNTR_DISABLED)
8442 continue;
8443
8444 if (port_cntrs[i].flags & CNTR_VL) {
8445 for (j = 0; j < C_VL_COUNT; j++) {
8446 memset(name, '\0', C_MAX_NAME);
8447 snprintf(name, C_MAX_NAME, "%s%d",
8448 port_cntrs[i].name,
8449 vl_from_idx(j));
8450 memcpy(p, name, strlen(name));
8451 p += strlen(name);
8452 *p++ = '\n';
8453 }
8454 } else {
8455 memcpy(p, port_cntrs[i].name,
8456 strlen(port_cntrs[i].name));
8457 p += strlen(port_cntrs[i].name);
8458 *p++ = '\n';
8459 }
8460 }
8461
8462 /* allocate per port storage for counter values */
8463 ppd = (struct hfi1_pportdata *)(dd + 1);
8464 for (i = 0; i < dd->num_pports; i++, ppd++) {
8465 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
8466 if (!ppd->cntrs)
8467 goto bail;
8468
8469 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
8470 if (!ppd->scntrs)
8471 goto bail;
8472 }
8473
8474 /* CPU counters need to be allocated and zeroed */
8475 if (init_cpu_counters(dd))
8476 goto bail;
8477
8478 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
8479 return 0;
8480bail:
8481 free_cntrs(dd);
8482 return -ENOMEM;
8483}
8484
8485
8486static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
8487{
8488 switch (chip_lstate) {
8489 default:
8490 dd_dev_err(dd,
8491 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
8492 chip_lstate);
8493 /* fall through */
8494 case LSTATE_DOWN:
8495 return IB_PORT_DOWN;
8496 case LSTATE_INIT:
8497 return IB_PORT_INIT;
8498 case LSTATE_ARMED:
8499 return IB_PORT_ARMED;
8500 case LSTATE_ACTIVE:
8501 return IB_PORT_ACTIVE;
8502 }
8503}
8504
8505u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
8506{
8507 /* look at the HFI meta-states only */
8508 switch (chip_pstate & 0xf0) {
8509 default:
8510 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
8511 chip_pstate);
8512 /* fall through */
8513 case PLS_DISABLED:
8514 return IB_PORTPHYSSTATE_DISABLED;
8515 case PLS_OFFLINE:
8516 return OPA_PORTPHYSSTATE_OFFLINE;
8517 case PLS_POLLING:
8518 return IB_PORTPHYSSTATE_POLLING;
8519 case PLS_CONFIGPHY:
8520 return IB_PORTPHYSSTATE_TRAINING;
8521 case PLS_LINKUP:
8522 return IB_PORTPHYSSTATE_LINKUP;
8523 case PLS_PHYTEST:
8524 return IB_PORTPHYSSTATE_PHY_TEST;
8525 }
8526}
8527
8528/* return the OPA port logical state name */
8529const char *opa_lstate_name(u32 lstate)
8530{
8531 static const char * const port_logical_names[] = {
8532 "PORT_NOP",
8533 "PORT_DOWN",
8534 "PORT_INIT",
8535 "PORT_ARMED",
8536 "PORT_ACTIVE",
8537 "PORT_ACTIVE_DEFER",
8538 };
8539 if (lstate < ARRAY_SIZE(port_logical_names))
8540 return port_logical_names[lstate];
8541 return "unknown";
8542}
8543
8544/* return the OPA port physical state name */
8545const char *opa_pstate_name(u32 pstate)
8546{
8547 static const char * const port_physical_names[] = {
8548 "PHYS_NOP",
8549 "reserved1",
8550 "PHYS_POLL",
8551 "PHYS_DISABLED",
8552 "PHYS_TRAINING",
8553 "PHYS_LINKUP",
8554 "PHYS_LINK_ERR_RECOVER",
8555 "PHYS_PHY_TEST",
8556 "reserved8",
8557 "PHYS_OFFLINE",
8558 "PHYS_GANGED",
8559 "PHYS_TEST",
8560 };
8561 if (pstate < ARRAY_SIZE(port_physical_names))
8562 return port_physical_names[pstate];
8563 return "unknown";
8564}
8565
8566/*
8567 * Read the hardware link state and set the driver's cached value of it.
8568 * Return the (new) current value.
8569 */
8570u32 get_logical_state(struct hfi1_pportdata *ppd)
8571{
8572 u32 new_state;
8573
8574 new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
8575 if (new_state != ppd->lstate) {
8576 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
8577 opa_lstate_name(new_state), new_state);
8578 ppd->lstate = new_state;
8579 }
8580 /*
8581 * Set port status flags in the page mapped into userspace
8582 * memory. Do it here to ensure a reliable state - this is
8583 * the only function called by all state handling code.
8584 * Always set the flags due to the fact that the cache value
8585 * might have been changed explicitly outside of this
8586 * function.
8587 */
8588 if (ppd->statusp) {
8589 switch (ppd->lstate) {
8590 case IB_PORT_DOWN:
8591 case IB_PORT_INIT:
8592 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
8593 HFI1_STATUS_IB_READY);
8594 break;
8595 case IB_PORT_ARMED:
8596 *ppd->statusp |= HFI1_STATUS_IB_CONF;
8597 break;
8598 case IB_PORT_ACTIVE:
8599 *ppd->statusp |= HFI1_STATUS_IB_READY;
8600 break;
8601 }
8602 }
8603 return ppd->lstate;
8604}
8605
8606/**
8607 * wait_logical_linkstate - wait for an IB link state change to occur
8608 * @ppd: port device
8609 * @state: the state to wait for
8610 * @msecs: the number of milliseconds to wait
8611 *
8612 * Wait up to msecs milliseconds for IB link state change to occur.
8613 * For now, take the easy polling route.
8614 * Returns 0 if state reached, otherwise -ETIMEDOUT.
8615 */
8616static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
8617 int msecs)
8618{
8619 unsigned long timeout;
8620
8621 timeout = jiffies + msecs_to_jiffies(msecs);
8622 while (1) {
8623 if (get_logical_state(ppd) == state)
8624 return 0;
8625 if (time_after(jiffies, timeout))
8626 break;
8627 msleep(20);
8628 }
8629 dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
8630
8631 return -ETIMEDOUT;
8632}
8633
8634u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
8635{
8636 static u32 remembered_state = 0xff;
8637 u32 pstate;
8638 u32 ib_pstate;
8639
8640 pstate = read_physical_state(ppd->dd);
8641 ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
8642 if (remembered_state != ib_pstate) {
8643 dd_dev_info(ppd->dd,
8644 "%s: physical state changed to %s (0x%x), phy 0x%x\n",
8645 __func__, opa_pstate_name(ib_pstate), ib_pstate,
8646 pstate);
8647 remembered_state = ib_pstate;
8648 }
8649 return ib_pstate;
8650}
8651
8652/*
8653 * Read/modify/write ASIC_QSFP register bits as selected by mask
8654 * data: 0 or 1 in the positions depending on what needs to be written
8655 * dir: 0 for read, 1 for write
8656 * mask: select by setting
8657 * I2CCLK (bit 0)
8658 * I2CDATA (bit 1)
8659 */
8660u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
8661 u32 mask)
8662{
8663 u64 qsfp_oe, target_oe;
8664
8665 target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
8666 if (mask) {
8667 /* We are writing register bits, so lock access */
8668 dir &= mask;
8669 data &= mask;
8670
8671 qsfp_oe = read_csr(dd, target_oe);
8672 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
8673 write_csr(dd, target_oe, qsfp_oe);
8674 }
8675 /* We are exclusively reading bits here, but it is unlikely
8676 * we'll get valid data when we set the direction of the pin
8677 * in the same call, so read should call this function again
8678 * to get valid data
8679 */
8680 return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
8681}
8682
8683#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
8684(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
8685
8686#define SET_STATIC_RATE_CONTROL_SMASK(r) \
8687(r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
8688
8689int hfi1_init_ctxt(struct send_context *sc)
8690{
8691 if (sc != NULL) {
8692 struct hfi1_devdata *dd = sc->dd;
8693 u64 reg;
8694 u8 set = (sc->type == SC_USER ?
8695 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
8696 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
8697 reg = read_kctxt_csr(dd, sc->hw_context,
8698 SEND_CTXT_CHECK_ENABLE);
8699 if (set)
8700 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
8701 else
8702 SET_STATIC_RATE_CONTROL_SMASK(reg);
8703 write_kctxt_csr(dd, sc->hw_context,
8704 SEND_CTXT_CHECK_ENABLE, reg);
8705 }
8706 return 0;
8707}
8708
8709int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
8710{
8711 int ret = 0;
8712 u64 reg;
8713
8714 if (dd->icode != ICODE_RTL_SILICON) {
8715 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
8716 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
8717 __func__);
8718 return -EINVAL;
8719 }
8720 reg = read_csr(dd, ASIC_STS_THERM);
8721 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
8722 ASIC_STS_THERM_CURR_TEMP_MASK);
8723 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
8724 ASIC_STS_THERM_LO_TEMP_MASK);
8725 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
8726 ASIC_STS_THERM_HI_TEMP_MASK);
8727 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
8728 ASIC_STS_THERM_CRIT_TEMP_MASK);
8729 /* triggers is a 3-bit value - 1 bit per trigger. */
8730 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
8731
8732 return ret;
8733}
8734
8735/* ========================================================================= */
8736
8737/*
8738 * Enable/disable chip from delivering interrupts.
8739 */
8740void set_intr_state(struct hfi1_devdata *dd, u32 enable)
8741{
8742 int i;
8743
8744 /*
8745 * In HFI, the mask needs to be 1 to allow interrupts.
8746 */
8747 if (enable) {
8748 u64 cce_int_mask;
8749 const int qsfp1_int_smask = QSFP1_INT % 64;
8750 const int qsfp2_int_smask = QSFP2_INT % 64;
8751
8752 /* enable all interrupts */
8753 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8754 write_csr(dd, CCE_INT_MASK + (8*i), ~(u64)0);
8755
8756 /*
8757 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
8758 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
8759 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
8760 * the index of the appropriate CSR in the CCEIntMask CSR array
8761 */
8762 cce_int_mask = read_csr(dd, CCE_INT_MASK +
8763 (8*(QSFP1_INT/64)));
8764 if (dd->hfi1_id) {
8765 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
8766 write_csr(dd, CCE_INT_MASK + (8*(QSFP1_INT/64)),
8767 cce_int_mask);
8768 } else {
8769 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
8770 write_csr(dd, CCE_INT_MASK + (8*(QSFP2_INT/64)),
8771 cce_int_mask);
8772 }
8773 } else {
8774 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8775 write_csr(dd, CCE_INT_MASK + (8*i), 0ull);
8776 }
8777}
8778
8779/*
8780 * Clear all interrupt sources on the chip.
8781 */
8782static void clear_all_interrupts(struct hfi1_devdata *dd)
8783{
8784 int i;
8785
8786 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8787 write_csr(dd, CCE_INT_CLEAR + (8*i), ~(u64)0);
8788
8789 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
8790 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
8791 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
8792 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
8793 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
8794 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
8795 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
8796 for (i = 0; i < dd->chip_send_contexts; i++)
8797 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
8798 for (i = 0; i < dd->chip_sdma_engines; i++)
8799 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
8800
8801 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
8802 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
8803 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
8804}
8805
8806/* Move to pcie.c? */
8807static void disable_intx(struct pci_dev *pdev)
8808{
8809 pci_intx(pdev, 0);
8810}
8811
8812static void clean_up_interrupts(struct hfi1_devdata *dd)
8813{
8814 int i;
8815
8816 /* remove irqs - must happen before disabling/turning off */
8817 if (dd->num_msix_entries) {
8818 /* MSI-X */
8819 struct hfi1_msix_entry *me = dd->msix_entries;
8820
8821 for (i = 0; i < dd->num_msix_entries; i++, me++) {
8822 if (me->arg == NULL) /* => no irq, no affinity */
8823 break;
8824 irq_set_affinity_hint(dd->msix_entries[i].msix.vector,
8825 NULL);
8826 free_irq(me->msix.vector, me->arg);
8827 }
8828 } else {
8829 /* INTx */
8830 if (dd->requested_intx_irq) {
8831 free_irq(dd->pcidev->irq, dd);
8832 dd->requested_intx_irq = 0;
8833 }
8834 }
8835
8836 /* turn off interrupts */
8837 if (dd->num_msix_entries) {
8838 /* MSI-X */
6e5b6131 8839 pci_disable_msix(dd->pcidev);
77241056
MM		 8840 } else {
8841 /* INTx */
8842 disable_intx(dd->pcidev);
8843 }
8844
8845 /* clean structures */
8846 for (i = 0; i < dd->num_msix_entries; i++)
8847 free_cpumask_var(dd->msix_entries[i].mask);
8848 kfree(dd->msix_entries);
8849 dd->msix_entries = NULL;
8850 dd->num_msix_entries = 0;
8851}
8852
8853/*
8854 * Remap the interrupt source from the general handler to the given MSI-X
8855 * interrupt.
8856 */
8857static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
8858{
8859 u64 reg;
8860 int m, n;
8861
8862 /* clear from the handled mask of the general interrupt */
8863 m = isrc / 64;
8864 n = isrc % 64;
8865 dd->gi_mask[m] &= ~((u64)1 << n);
8866
8867 /* direct the chip source to the given MSI-X interrupt */
8868 m = isrc / 8;
8869 n = isrc % 8;
8870 reg = read_csr(dd, CCE_INT_MAP + (8*m));
8871 reg &= ~((u64)0xff << (8*n));
8872 reg |= ((u64)msix_intr & 0xff) << (8*n);
8873 write_csr(dd, CCE_INT_MAP + (8*m), reg);
8874}
8875
8876static void remap_sdma_interrupts(struct hfi1_devdata *dd,
8877 int engine, int msix_intr)
8878{
8879 /*
8880 * SDMA engine interrupt sources grouped by type, rather than
8881 * engine. Per-engine interrupts are as follows:
8882 * SDMA
8883 * SDMAProgress
8884 * SDMAIdle
8885 */
8886 remap_intr(dd, IS_SDMA_START + 0*TXE_NUM_SDMA_ENGINES + engine,
8887 msix_intr);
8888 remap_intr(dd, IS_SDMA_START + 1*TXE_NUM_SDMA_ENGINES + engine,
8889 msix_intr);
8890 remap_intr(dd, IS_SDMA_START + 2*TXE_NUM_SDMA_ENGINES + engine,
8891 msix_intr);
8892}
8893
77241056
MM		 8894static int request_intx_irq(struct hfi1_devdata *dd)
8895{
8896 int ret;
8897
9805071e
JJ		 8898 snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
8899 dd->unit);
77241056
MM		 8900 ret = request_irq(dd->pcidev->irq, general_interrupt,
8901 IRQF_SHARED, dd->intx_name, dd);
8902 if (ret)
8903 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
8904 ret);
8905 else
8906 dd->requested_intx_irq = 1;
8907 return ret;
8908}
8909
8910static int request_msix_irqs(struct hfi1_devdata *dd)
8911{
8912 const struct cpumask *local_mask;
8913 cpumask_var_t def, rcv;
8914 bool def_ret, rcv_ret;
8915 int first_general, last_general;
8916 int first_sdma, last_sdma;
8917 int first_rx, last_rx;
82c2611d 8918 int first_cpu, curr_cpu;
77241056
MM		 8919 int rcv_cpu, sdma_cpu;
8920 int i, ret = 0, possible;
8921 int ht;
8922
8923 /* calculate the ranges we are going to use */
8924 first_general = 0;
8925 first_sdma = last_general = first_general + 1;
8926 first_rx = last_sdma = first_sdma + dd->num_sdma;
8927 last_rx = first_rx + dd->n_krcv_queues;
8928
8929 /*
8930 * Interrupt affinity.
8931 *
8932 * non-rcv avail gets a default mask that
8933 * starts as possible cpus with threads reset
8934 * and each rcv avail reset.
8935 *
8936 * rcv avail gets node relative 1 wrapping back
8937 * to the node relative 1 as necessary.
8938 *
8939 */
8940 local_mask = cpumask_of_pcibus(dd->pcidev->bus);
8941 /* if first cpu is invalid, use NUMA 0 */
8942 if (cpumask_first(local_mask) >= nr_cpu_ids)
8943 local_mask = topology_core_cpumask(0);
8944
8945 def_ret = zalloc_cpumask_var(&def, GFP_KERNEL);
8946 rcv_ret = zalloc_cpumask_var(&rcv, GFP_KERNEL);
8947 if (!def_ret || !rcv_ret)
8948 goto bail;
8949 /* use local mask as default */
8950 cpumask_copy(def, local_mask);
8951 possible = cpumask_weight(def);
8952 /* disarm threads from default */
8953 ht = cpumask_weight(
8954 topology_sibling_cpumask(cpumask_first(local_mask)));
8955 for (i = possible/ht; i < possible; i++)
8956 cpumask_clear_cpu(i, def);
77241056
MM		 8957 /* def now has full cores on chosen node*/
8958 first_cpu = cpumask_first(def);
8959 if (nr_cpu_ids >= first_cpu)
8960 first_cpu++;
82c2611d 8961 curr_cpu = first_cpu;
77241056 8962
82c2611d
NV		 8963 /* One context is reserved as control context */
8964 for (i = first_cpu; i < dd->n_krcv_queues + first_cpu - 1; i++) {
77241056
MM		 8965 cpumask_clear_cpu(curr_cpu, def);
8966 cpumask_set_cpu(curr_cpu, rcv);
82c2611d
NV		 8967 curr_cpu = cpumask_next(curr_cpu, def);
8968 if (curr_cpu >= nr_cpu_ids)
8969 break;
77241056
MM		 8970 }
8971 /* def mask has non-rcv, rcv has recv mask */
8972 rcv_cpu = cpumask_first(rcv);
8973 sdma_cpu = cpumask_first(def);
8974
8975 /*
8976 * Sanity check - the code expects all SDMA chip source
8977 * interrupts to be in the same CSR, starting at bit 0. Verify
8978 * that this is true by checking the bit location of the start.
8979 */
8980 BUILD_BUG_ON(IS_SDMA_START % 64);
8981
8982 for (i = 0; i < dd->num_msix_entries; i++) {
8983 struct hfi1_msix_entry *me = &dd->msix_entries[i];
8984 const char *err_info;
8985 irq_handler_t handler;
f4f30031 8986 irq_handler_t thread = NULL;
77241056
MM		 8987 void *arg;
8988 int idx;
8989 struct hfi1_ctxtdata *rcd = NULL;
8990 struct sdma_engine *sde = NULL;
8991
8992 /* obtain the arguments to request_irq */
8993 if (first_general <= i && i < last_general) {
8994 idx = i - first_general;
8995 handler = general_interrupt;
8996 arg = dd;
8997 snprintf(me->name, sizeof(me->name),
9805071e 8998 DRIVER_NAME "_%d", dd->unit);
77241056
MM		 8999 err_info = "general";
9000 } else if (first_sdma <= i && i < last_sdma) {
9001 idx = i - first_sdma;
9002 sde = &dd->per_sdma[idx];
9003 handler = sdma_interrupt;
9004 arg = sde;
9005 snprintf(me->name, sizeof(me->name),
9805071e 9006 DRIVER_NAME "_%d sdma%d", dd->unit, idx);
77241056
MM		 9007 err_info = "sdma";
9008 remap_sdma_interrupts(dd, idx, i);
9009 } else if (first_rx <= i && i < last_rx) {
9010 idx = i - first_rx;
9011 rcd = dd->rcd[idx];
9012 /* no interrupt if no rcd */
9013 if (!rcd)
9014 continue;
9015 /*
9016 * Set the interrupt register and mask for this
9017 * context's interrupt.
9018 */
9019 rcd->ireg = (IS_RCVAVAIL_START+idx) / 64;
9020 rcd->imask = ((u64)1) <<
9021 ((IS_RCVAVAIL_START+idx) % 64);
9022 handler = receive_context_interrupt;
f4f30031 9023 thread = receive_context_thread;
77241056
MM		 9024 arg = rcd;
9025 snprintf(me->name, sizeof(me->name),
9805071e 9026 DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
77241056 9027 err_info = "receive context";
66c0933b 9028 remap_intr(dd, IS_RCVAVAIL_START + idx, i);
77241056
MM		 9029 } else {
9030 /* not in our expected range - complain, then
9031 ignore it */
9032 dd_dev_err(dd,
9033 "Unexpected extra MSI-X interrupt %d\n", i);
9034 continue;
9035 }
9036 /* no argument, no interrupt */
9037 if (arg == NULL)
9038 continue;
9039 /* make sure the name is terminated */
9040 me->name[sizeof(me->name)-1] = 0;
9041
f4f30031
DL		 9042 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
9043 me->name, arg);
77241056
MM		 9044 if (ret) {
9045 dd_dev_err(dd,
9046 "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
9047 err_info, me->msix.vector, idx, ret);
9048 return ret;
9049 }
9050 /*
9051 * assign arg after request_irq call, so it will be
9052 * cleaned up
9053 */
9054 me->arg = arg;
9055
9056 if (!zalloc_cpumask_var(
9057 &dd->msix_entries[i].mask,
9058 GFP_KERNEL))
9059 goto bail;
9060 if (handler == sdma_interrupt) {
9061 dd_dev_info(dd, "sdma engine %d cpu %d\n",
9062 sde->this_idx, sdma_cpu);
0a226edd 9063 sde->cpu = sdma_cpu;
77241056
MM		 9064 cpumask_set_cpu(sdma_cpu, dd->msix_entries[i].mask);
9065 sdma_cpu = cpumask_next(sdma_cpu, def);
9066 if (sdma_cpu >= nr_cpu_ids)
9067 sdma_cpu = cpumask_first(def);
9068 } else if (handler == receive_context_interrupt) {
82c2611d
NV		 9069 dd_dev_info(dd, "rcv ctxt %d cpu %d\n", rcd->ctxt,
9070 (rcd->ctxt == HFI1_CTRL_CTXT) ?
9071 cpumask_first(def) : rcv_cpu);
9072 if (rcd->ctxt == HFI1_CTRL_CTXT) {
9073 /* map to first default */
9074 cpumask_set_cpu(cpumask_first(def),
9075 dd->msix_entries[i].mask);
9076 } else {
9077 cpumask_set_cpu(rcv_cpu,
9078 dd->msix_entries[i].mask);
9079 rcv_cpu = cpumask_next(rcv_cpu, rcv);
9080 if (rcv_cpu >= nr_cpu_ids)
9081 rcv_cpu = cpumask_first(rcv);
9082 }
77241056
MM		 9083 } else {
9084 /* otherwise first def */
9085 dd_dev_info(dd, "%s cpu %d\n",
9086 err_info, cpumask_first(def));
9087 cpumask_set_cpu(
9088 cpumask_first(def), dd->msix_entries[i].mask);
9089 }
9090 irq_set_affinity_hint(
9091 dd->msix_entries[i].msix.vector,
9092 dd->msix_entries[i].mask);
9093 }
9094
9095out:
9096 free_cpumask_var(def);
9097 free_cpumask_var(rcv);
9098 return ret;
9099bail:
9100 ret = -ENOMEM;
9101 goto out;
9102}
9103
9104/*
9105 * Set the general handler to accept all interrupts, remap all
9106 * chip interrupts back to MSI-X 0.
9107 */
9108static void reset_interrupts(struct hfi1_devdata *dd)
9109{
9110 int i;
9111
9112 /* all interrupts handled by the general handler */
9113 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
9114 dd->gi_mask[i] = ~(u64)0;
9115
9116 /* all chip interrupts map to MSI-X 0 */
9117 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9118 write_csr(dd, CCE_INT_MAP + (8*i), 0);
9119}
9120
9121static int set_up_interrupts(struct hfi1_devdata *dd)
9122{
9123 struct hfi1_msix_entry *entries;
9124 u32 total, request;
9125 int i, ret;
9126 int single_interrupt = 0; /* we expect to have all the interrupts */
9127
9128 /*
9129 * Interrupt count:
9130 * 1 general, "slow path" interrupt (includes the SDMA engines
9131 * slow source, SDMACleanupDone)
9132 * N interrupts - one per used SDMA engine
9133 * M interrupt - one per kernel receive context
9134 */
9135 total = 1 + dd->num_sdma + dd->n_krcv_queues;
9136
9137 entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
9138 if (!entries) {
77241056
MM		 9139 ret = -ENOMEM;
9140 goto fail;
9141 }
9142 /* 1-1 MSI-X entry assignment */
9143 for (i = 0; i < total; i++)
9144 entries[i].msix.entry = i;
9145
9146 /* ask for MSI-X interrupts */
9147 request = total;
9148 request_msix(dd, &request, entries);
9149
9150 if (request == 0) {
9151 /* using INTx */
9152 /* dd->num_msix_entries already zero */
9153 kfree(entries);
9154 single_interrupt = 1;
9155 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
9156 } else {
9157 /* using MSI-X */
9158 dd->num_msix_entries = request;
9159 dd->msix_entries = entries;
9160
9161 if (request != total) {
9162 /* using MSI-X, with reduced interrupts */
9163 dd_dev_err(
9164 dd,
9165 "cannot handle reduced interrupt case, want %u, got %u\n",
9166 total, request);
9167 ret = -EINVAL;
9168 goto fail;
9169 }
9170 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
9171 }
9172
9173 /* mask all interrupts */
9174 set_intr_state(dd, 0);
9175 /* clear all pending interrupts */
9176 clear_all_interrupts(dd);
9177
9178 /* reset general handler mask, chip MSI-X mappings */
9179 reset_interrupts(dd);
9180
9181 if (single_interrupt)
9182 ret = request_intx_irq(dd);
9183 else
9184 ret = request_msix_irqs(dd);
9185 if (ret)
9186 goto fail;
9187
9188 return 0;
9189
9190fail:
9191 clean_up_interrupts(dd);
9192 return ret;
9193}
9194
9195/*
9196 * Set up context values in dd. Sets:
9197 *
9198 * num_rcv_contexts - number of contexts being used
9199 * n_krcv_queues - number of kernel contexts
9200 * first_user_ctxt - first non-kernel context in array of contexts
9201 * freectxts - number of free user contexts
9202 * num_send_contexts - number of PIO send contexts being used
9203 */
9204static int set_up_context_variables(struct hfi1_devdata *dd)
9205{
9206 int num_kernel_contexts;
9207 int num_user_contexts;
9208 int total_contexts;
9209 int ret;
9210 unsigned ngroups;
9211
9212 /*
9213 * Kernel contexts: (to be fixed later):
9214 * - min or 2 or 1 context/numa
82c2611d
NV		 9215 * - Context 0 - control context (VL15/multicast/error)
9216 * - Context 1 - default context
77241056
MM		 9217 */
9218 if (n_krcvqs)
82c2611d
NV		 9219 /*
9220 * Don't count context 0 in n_krcvqs since
9221 * is isn't used for normal verbs traffic.
9222 *
9223 * krcvqs will reflect number of kernel
9224 * receive contexts above 0.
9225 */
9226 num_kernel_contexts = n_krcvqs + MIN_KERNEL_KCTXTS - 1;
77241056
MM		 9227 else
9228 num_kernel_contexts = num_online_nodes();
9229 num_kernel_contexts =
9230 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
9231 /*
9232 * Every kernel receive context needs an ACK send context.
9233 * one send context is allocated for each VL{0-7} and VL15
9234 */
9235 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
9236 dd_dev_err(dd,
9237 "Reducing # kernel rcv contexts to: %d, from %d\n",
9238 (int)(dd->chip_send_contexts - num_vls - 1),
9239 (int)num_kernel_contexts);
9240 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
9241 }
9242 /*
9243 * User contexts: (to be fixed later)
9244 * - set to num_rcv_contexts if non-zero
9245 * - default to 1 user context per CPU
9246 */
9247 if (num_rcv_contexts)
9248 num_user_contexts = num_rcv_contexts;
9249 else
9250 num_user_contexts = num_online_cpus();
9251
9252 total_contexts = num_kernel_contexts + num_user_contexts;
9253
9254 /*
9255 * Adjust the counts given a global max.
9256 */
9257 if (total_contexts > dd->chip_rcv_contexts) {
9258 dd_dev_err(dd,
9259 "Reducing # user receive contexts to: %d, from %d\n",
9260 (int)(dd->chip_rcv_contexts - num_kernel_contexts),
9261 (int)num_user_contexts);
9262 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
9263 /* recalculate */
9264 total_contexts = num_kernel_contexts + num_user_contexts;
9265 }
9266
9267 /* the first N are kernel contexts, the rest are user contexts */
9268 dd->num_rcv_contexts = total_contexts;
9269 dd->n_krcv_queues = num_kernel_contexts;
9270 dd->first_user_ctxt = num_kernel_contexts;
9271 dd->freectxts = num_user_contexts;
9272 dd_dev_info(dd,
9273 "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
9274 (int)dd->chip_rcv_contexts,
9275 (int)dd->num_rcv_contexts,
9276 (int)dd->n_krcv_queues,
9277 (int)dd->num_rcv_contexts - dd->n_krcv_queues);
9278
9279 /*
9280 * Receive array allocation:
9281 * All RcvArray entries are divided into groups of 8. This
9282 * is required by the hardware and will speed up writes to
9283 * consecutive entries by using write-combining of the entire
9284 * cacheline.
9285 *
9286 * The number of groups are evenly divided among all contexts.
9287 * any left over groups will be given to the first N user
9288 * contexts.
9289 */
9290 dd->rcv_entries.group_size = RCV_INCREMENT;
9291 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
9292 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
9293 dd->rcv_entries.nctxt_extra = ngroups -
9294 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
9295 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
9296 dd->rcv_entries.ngroups,
9297 dd->rcv_entries.nctxt_extra);
9298 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
9299 MAX_EAGER_ENTRIES * 2) {
9300 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
9301 dd->rcv_entries.group_size;
9302 dd_dev_info(dd,
9303 "RcvArray group count too high, change to %u\n",
9304 dd->rcv_entries.ngroups);
9305 dd->rcv_entries.nctxt_extra = 0;
9306 }
9307 /*
9308 * PIO send contexts
9309 */
9310 ret = init_sc_pools_and_sizes(dd);
9311 if (ret >= 0) { /* success */
9312 dd->num_send_contexts = ret;
9313 dd_dev_info(
9314 dd,
9315 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d)\n",
9316 dd->chip_send_contexts,
9317 dd->num_send_contexts,
9318 dd->sc_sizes[SC_KERNEL].count,
9319 dd->sc_sizes[SC_ACK].count,
9320 dd->sc_sizes[SC_USER].count);
9321 ret = 0; /* success */
9322 }
9323
9324 return ret;
9325}
9326
9327/*
9328 * Set the device/port partition key table. The MAD code
9329 * will ensure that, at least, the partial management
9330 * partition key is present in the table.
9331 */
9332static void set_partition_keys(struct hfi1_pportdata *ppd)
9333{
9334 struct hfi1_devdata *dd = ppd->dd;
9335 u64 reg = 0;
9336 int i;
9337
9338 dd_dev_info(dd, "Setting partition keys\n");
9339 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
9340 reg |= (ppd->pkeys[i] &
9341 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
9342 ((i % 4) *
9343 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
9344 /* Each register holds 4 PKey values. */
9345 if ((i % 4) == 3) {
9346 write_csr(dd, RCV_PARTITION_KEY +
9347 ((i - 3) * 2), reg);
9348 reg = 0;
9349 }
9350 }
9351
9352 /* Always enable HW pkeys check when pkeys table is set */
9353 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
9354}
9355
9356/*
9357 * These CSRs and memories are uninitialized on reset and must be
9358 * written before reading to set the ECC/parity bits.
9359 *
9360 * NOTE: All user context CSRs that are not mmaped write-only
9361 * (e.g. the TID flows) must be initialized even if the driver never
9362 * reads them.
9363 */
9364static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
9365{
9366 int i, j;
9367
9368 /* CceIntMap */
9369 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9370 write_csr(dd, CCE_INT_MAP+(8*i), 0);
9371
9372 /* SendCtxtCreditReturnAddr */
9373 for (i = 0; i < dd->chip_send_contexts; i++)
9374 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
9375
9376 /* PIO Send buffers */
9377 /* SDMA Send buffers */
9378 /* These are not normally read, and (presently) have no method
9379 to be read, so are not pre-initialized */
9380
9381 /* RcvHdrAddr */
9382 /* RcvHdrTailAddr */
9383 /* RcvTidFlowTable */
9384 for (i = 0; i < dd->chip_rcv_contexts; i++) {
9385 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
9386 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
9387 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
9388 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE+(8*j), 0);
9389 }
9390
9391 /* RcvArray */
9392 for (i = 0; i < dd->chip_rcv_array_count; i++)
9393 write_csr(dd, RCV_ARRAY + (8*i),
9394 RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
9395
9396 /* RcvQPMapTable */
9397 for (i = 0; i < 32; i++)
9398 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
9399}
9400
9401/*
9402 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
9403 */
9404static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
9405 u64 ctrl_bits)
9406{
9407 unsigned long timeout;
9408 u64 reg;
9409
9410 /* is the condition present? */
9411 reg = read_csr(dd, CCE_STATUS);
9412 if ((reg & status_bits) == 0)
9413 return;
9414
9415 /* clear the condition */
9416 write_csr(dd, CCE_CTRL, ctrl_bits);
9417
9418 /* wait for the condition to clear */
9419 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
9420 while (1) {
9421 reg = read_csr(dd, CCE_STATUS);
9422 if ((reg & status_bits) == 0)
9423 return;
9424 if (time_after(jiffies, timeout)) {
9425 dd_dev_err(dd,
9426 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
9427 status_bits, reg & status_bits);
9428 return;
9429 }
9430 udelay(1);
9431 }
9432}
9433
9434/* set CCE CSRs to chip reset defaults */
9435static void reset_cce_csrs(struct hfi1_devdata *dd)
9436{
9437 int i;
9438
9439 /* CCE_REVISION read-only */
9440 /* CCE_REVISION2 read-only */
9441 /* CCE_CTRL - bits clear automatically */
9442 /* CCE_STATUS read-only, use CceCtrl to clear */
9443 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
9444 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
9445 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
9446 for (i = 0; i < CCE_NUM_SCRATCH; i++)
9447 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
9448 /* CCE_ERR_STATUS read-only */
9449 write_csr(dd, CCE_ERR_MASK, 0);
9450 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
9451 /* CCE_ERR_FORCE leave alone */
9452 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
9453 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
9454 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
9455 /* CCE_PCIE_CTRL leave alone */
9456 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
9457 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
9458 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
9459 CCE_MSIX_TABLE_UPPER_RESETCSR);
9460 }
9461 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
9462 /* CCE_MSIX_PBA read-only */
9463 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
9464 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
9465 }
9466 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9467 write_csr(dd, CCE_INT_MAP, 0);
9468 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
9469 /* CCE_INT_STATUS read-only */
9470 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
9471 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
9472 /* CCE_INT_FORCE leave alone */
9473 /* CCE_INT_BLOCKED read-only */
9474 }
9475 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
9476 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
9477}
9478
9479/* set ASIC CSRs to chip reset defaults */
9480static void reset_asic_csrs(struct hfi1_devdata *dd)
9481{
77241056
MM		 9482 int i;
9483
9484 /*
9485 * If the HFIs are shared between separate nodes or VMs,
9486 * then more will need to be done here. One idea is a module
9487 * parameter that returns early, letting the first power-on or
9488 * a known first load do the reset and blocking all others.
9489 */
9490
7c03ed85
EH		 9491 if (!(dd->flags & HFI1_DO_INIT_ASIC))
9492 return;
77241056
MM		 9493
9494 if (dd->icode != ICODE_FPGA_EMULATION) {
9495 /* emulation does not have an SBus - leave these alone */
9496 /*
9497 * All writes to ASIC_CFG_SBUS_REQUEST do something.
9498 * Notes:
9499 * o The reset is not zero if aimed at the core. See the
9500 * SBus documentation for details.
9501 * o If the SBus firmware has been updated (e.g. by the BIOS),
9502 * will the reset revert that?
9503 */
9504 /* ASIC_CFG_SBUS_REQUEST leave alone */
9505 write_csr(dd, ASIC_CFG_SBUS_EXECUTE, 0);
9506 }
9507 /* ASIC_SBUS_RESULT read-only */
9508 write_csr(dd, ASIC_STS_SBUS_COUNTERS, 0);
9509 for (i = 0; i < ASIC_NUM_SCRATCH; i++)
9510 write_csr(dd, ASIC_CFG_SCRATCH + (8 * i), 0);
9511 write_csr(dd, ASIC_CFG_MUTEX, 0); /* this will clear it */
7c03ed85
EH		 9512
9513 /* We might want to retain this state across FLR if we ever use it */
77241056 9514 write_csr(dd, ASIC_CFG_DRV_STR, 0);
7c03ed85 9515
4ef98989 9516 /* ASIC_CFG_THERM_POLL_EN leave alone */
77241056
MM		 9517 /* ASIC_STS_THERM read-only */
9518 /* ASIC_CFG_RESET leave alone */
9519
9520 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, 0);
9521 /* ASIC_PCIE_SD_HOST_STATUS read-only */
9522 write_csr(dd, ASIC_PCIE_SD_INTRPT_DATA_CODE, 0);
9523 write_csr(dd, ASIC_PCIE_SD_INTRPT_ENABLE, 0);
9524 /* ASIC_PCIE_SD_INTRPT_PROGRESS read-only */
9525 write_csr(dd, ASIC_PCIE_SD_INTRPT_STATUS, ~0ull); /* clear */
9526 /* ASIC_HFI0_PCIE_SD_INTRPT_RSPD_DATA read-only */
9527 /* ASIC_HFI1_PCIE_SD_INTRPT_RSPD_DATA read-only */
9528 for (i = 0; i < 16; i++)
9529 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (8 * i), 0);
9530
9531 /* ASIC_GPIO_IN read-only */
9532 write_csr(dd, ASIC_GPIO_OE, 0);
9533 write_csr(dd, ASIC_GPIO_INVERT, 0);
9534 write_csr(dd, ASIC_GPIO_OUT, 0);
9535 write_csr(dd, ASIC_GPIO_MASK, 0);
9536 /* ASIC_GPIO_STATUS read-only */
9537 write_csr(dd, ASIC_GPIO_CLEAR, ~0ull);
9538 /* ASIC_GPIO_FORCE leave alone */
9539
9540 /* ASIC_QSFP1_IN read-only */
9541 write_csr(dd, ASIC_QSFP1_OE, 0);
9542 write_csr(dd, ASIC_QSFP1_INVERT, 0);
9543 write_csr(dd, ASIC_QSFP1_OUT, 0);
9544 write_csr(dd, ASIC_QSFP1_MASK, 0);
9545 /* ASIC_QSFP1_STATUS read-only */
9546 write_csr(dd, ASIC_QSFP1_CLEAR, ~0ull);
9547 /* ASIC_QSFP1_FORCE leave alone */
9548
9549 /* ASIC_QSFP2_IN read-only */
9550 write_csr(dd, ASIC_QSFP2_OE, 0);
9551 write_csr(dd, ASIC_QSFP2_INVERT, 0);
9552 write_csr(dd, ASIC_QSFP2_OUT, 0);
9553 write_csr(dd, ASIC_QSFP2_MASK, 0);
9554 /* ASIC_QSFP2_STATUS read-only */
9555 write_csr(dd, ASIC_QSFP2_CLEAR, ~0ull);
9556 /* ASIC_QSFP2_FORCE leave alone */
9557
9558 write_csr(dd, ASIC_EEP_CTL_STAT, ASIC_EEP_CTL_STAT_RESETCSR);
9559 /* this also writes a NOP command, clearing paging mode */
9560 write_csr(dd, ASIC_EEP_ADDR_CMD, 0);
9561 write_csr(dd, ASIC_EEP_DATA, 0);
77241056
MM		 9562}
9563
9564/* set MISC CSRs to chip reset defaults */
9565static void reset_misc_csrs(struct hfi1_devdata *dd)
9566{
9567 int i;
9568
9569 for (i = 0; i < 32; i++) {
9570 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
9571 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
9572 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
9573 }
9574 /* MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
9575 only be written 128-byte chunks */
9576 /* init RSA engine to clear lingering errors */
9577 write_csr(dd, MISC_CFG_RSA_CMD, 1);
9578 write_csr(dd, MISC_CFG_RSA_MU, 0);
9579 write_csr(dd, MISC_CFG_FW_CTRL, 0);
9580 /* MISC_STS_8051_DIGEST read-only */
9581 /* MISC_STS_SBM_DIGEST read-only */
9582 /* MISC_STS_PCIE_DIGEST read-only */
9583 /* MISC_STS_FAB_DIGEST read-only */
9584 /* MISC_ERR_STATUS read-only */
9585 write_csr(dd, MISC_ERR_MASK, 0);
9586 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
9587 /* MISC_ERR_FORCE leave alone */
9588}
9589
9590/* set TXE CSRs to chip reset defaults */
9591static void reset_txe_csrs(struct hfi1_devdata *dd)
9592{
9593 int i;
9594
9595 /*
9596 * TXE Kernel CSRs
9597 */
9598 write_csr(dd, SEND_CTRL, 0);
9599 __cm_reset(dd, 0); /* reset CM internal state */
9600 /* SEND_CONTEXTS read-only */
9601 /* SEND_DMA_ENGINES read-only */
9602 /* SEND_PIO_MEM_SIZE read-only */
9603 /* SEND_DMA_MEM_SIZE read-only */
9604 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
9605 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
9606 /* SEND_PIO_ERR_STATUS read-only */
9607 write_csr(dd, SEND_PIO_ERR_MASK, 0);
9608 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
9609 /* SEND_PIO_ERR_FORCE leave alone */
9610 /* SEND_DMA_ERR_STATUS read-only */
9611 write_csr(dd, SEND_DMA_ERR_MASK, 0);
9612 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
9613 /* SEND_DMA_ERR_FORCE leave alone */
9614 /* SEND_EGRESS_ERR_STATUS read-only */
9615 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
9616 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
9617 /* SEND_EGRESS_ERR_FORCE leave alone */
9618 write_csr(dd, SEND_BTH_QP, 0);
9619 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
9620 write_csr(dd, SEND_SC2VLT0, 0);
9621 write_csr(dd, SEND_SC2VLT1, 0);
9622 write_csr(dd, SEND_SC2VLT2, 0);
9623 write_csr(dd, SEND_SC2VLT3, 0);
9624 write_csr(dd, SEND_LEN_CHECK0, 0);
9625 write_csr(dd, SEND_LEN_CHECK1, 0);
9626 /* SEND_ERR_STATUS read-only */
9627 write_csr(dd, SEND_ERR_MASK, 0);
9628 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
9629 /* SEND_ERR_FORCE read-only */
9630 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
9631 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8*i), 0);
9632 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
9633 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8*i), 0);
9634 for (i = 0; i < dd->chip_send_contexts/NUM_CONTEXTS_PER_SET; i++)
9635 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8*i), 0);
9636 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
9637 write_csr(dd, SEND_COUNTER_ARRAY32 + (8*i), 0);
9638 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
9639 write_csr(dd, SEND_COUNTER_ARRAY64 + (8*i), 0);
9640 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
9641 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
9642 SEND_CM_GLOBAL_CREDIT_RESETCSR);
9643 /* SEND_CM_CREDIT_USED_STATUS read-only */
9644 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
9645 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
9646 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
9647 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
9648 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
9649 for (i = 0; i < TXE_NUM_DATA_VL; i++)
9650 write_csr(dd, SEND_CM_CREDIT_VL + (8*i), 0);
9651 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
9652 /* SEND_CM_CREDIT_USED_VL read-only */
9653 /* SEND_CM_CREDIT_USED_VL15 read-only */
9654 /* SEND_EGRESS_CTXT_STATUS read-only */
9655 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
9656 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
9657 /* SEND_EGRESS_ERR_INFO read-only */
9658 /* SEND_EGRESS_ERR_SOURCE read-only */
9659
9660 /*
9661 * TXE Per-Context CSRs
9662 */
9663 for (i = 0; i < dd->chip_send_contexts; i++) {
9664 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
9665 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
9666 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
9667 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
9668 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
9669 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
9670 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
9671 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
9672 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
9673 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
9674 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
9675 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
9676 }
9677
9678 /*
9679 * TXE Per-SDMA CSRs
9680 */
9681 for (i = 0; i < dd->chip_sdma_engines; i++) {
9682 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
9683 /* SEND_DMA_STATUS read-only */
9684 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
9685 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
9686 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
9687 /* SEND_DMA_HEAD read-only */
9688 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
9689 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
9690 /* SEND_DMA_IDLE_CNT read-only */
9691 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
9692 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
9693 /* SEND_DMA_DESC_FETCHED_CNT read-only */
9694 /* SEND_DMA_ENG_ERR_STATUS read-only */
9695 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
9696 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
9697 /* SEND_DMA_ENG_ERR_FORCE leave alone */
9698 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
9699 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
9700 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
9701 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
9702 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
9703 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
9704 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
9705 }
9706}
9707
9708/*
9709 * Expect on entry:
9710 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
9711 */
9712static void init_rbufs(struct hfi1_devdata *dd)
9713{
9714 u64 reg;
9715 int count;
9716
9717 /*
9718 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
9719 * clear.
9720 */
9721 count = 0;
9722 while (1) {
9723 reg = read_csr(dd, RCV_STATUS);
9724 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
9725 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
9726 break;
9727 /*
9728 * Give up after 1ms - maximum wait time.
9729 *
9730 * RBuf size is 148KiB. Slowest possible is PCIe Gen1 x1 at
9731 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
9732 * 148 KB / (66% * 250MB/s) = 920us
9733 */
9734 if (count++ > 500) {
9735 dd_dev_err(dd,
9736 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
9737 __func__, reg);
9738 break;
9739 }
9740 udelay(2); /* do not busy-wait the CSR */
9741 }
9742
9743 /* start the init - expect RcvCtrl to be 0 */
9744 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
9745
9746 /*
9747 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
9748 * period after the write before RcvStatus.RxRbufInitDone is valid.
9749 * The delay in the first run through the loop below is sufficient and
9750 * required before the first read of RcvStatus.RxRbufInintDone.
9751 */
9752 read_csr(dd, RCV_CTRL);
9753
9754 /* wait for the init to finish */
9755 count = 0;
9756 while (1) {
9757 /* delay is required first time through - see above */
9758 udelay(2); /* do not busy-wait the CSR */
9759 reg = read_csr(dd, RCV_STATUS);
9760 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
9761 break;
9762
9763 /* give up after 100us - slowest possible at 33MHz is 73us */
9764 if (count++ > 50) {
9765 dd_dev_err(dd,
9766 "%s: RcvStatus.RxRbufInit not set, continuing\n",
9767 __func__);
9768 break;
9769 }
9770 }
9771}
9772
9773/* set RXE CSRs to chip reset defaults */
9774static void reset_rxe_csrs(struct hfi1_devdata *dd)
9775{
9776 int i, j;
9777
9778 /*
9779 * RXE Kernel CSRs
9780 */
9781 write_csr(dd, RCV_CTRL, 0);
9782 init_rbufs(dd);
9783 /* RCV_STATUS read-only */
9784 /* RCV_CONTEXTS read-only */
9785 /* RCV_ARRAY_CNT read-only */
9786 /* RCV_BUF_SIZE read-only */
9787 write_csr(dd, RCV_BTH_QP, 0);
9788 write_csr(dd, RCV_MULTICAST, 0);
9789 write_csr(dd, RCV_BYPASS, 0);
9790 write_csr(dd, RCV_VL15, 0);
9791 /* this is a clear-down */
9792 write_csr(dd, RCV_ERR_INFO,
9793 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
9794 /* RCV_ERR_STATUS read-only */
9795 write_csr(dd, RCV_ERR_MASK, 0);
9796 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
9797 /* RCV_ERR_FORCE leave alone */
9798 for (i = 0; i < 32; i++)
9799 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
9800 for (i = 0; i < 4; i++)
9801 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
9802 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
9803 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
9804 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
9805 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
9806 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
9807 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
9808 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
9809 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
9810 }
9811 for (i = 0; i < 32; i++)
9812 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
9813
9814 /*
9815 * RXE Kernel and User Per-Context CSRs
9816 */
9817 for (i = 0; i < dd->chip_rcv_contexts; i++) {
9818 /* kernel */
9819 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
9820 /* RCV_CTXT_STATUS read-only */
9821 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
9822 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
9823 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
9824 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
9825 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
9826 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
9827 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
9828 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
9829 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
9830 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
9831
9832 /* user */
9833 /* RCV_HDR_TAIL read-only */
9834 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
9835 /* RCV_EGR_INDEX_TAIL read-only */
9836 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
9837 /* RCV_EGR_OFFSET_TAIL read-only */
9838 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
9839 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j),
9840 0);
9841 }
9842 }
9843}
9844
9845/*
9846 * Set sc2vl tables.
9847 *
9848 * They power on to zeros, so to avoid send context errors
9849 * they need to be set:
9850 *
9851 * SC 0-7 -> VL 0-7 (respectively)
9852 * SC 15 -> VL 15
9853 * otherwise
9854 * -> VL 0
9855 */
9856static void init_sc2vl_tables(struct hfi1_devdata *dd)
9857{
9858 int i;
9859 /* init per architecture spec, constrained by hardware capability */
9860
9861 /* HFI maps sent packets */
9862 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
9863 0,
9864 0, 0, 1, 1,
9865 2, 2, 3, 3,
9866 4, 4, 5, 5,
9867 6, 6, 7, 7));
9868 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
9869 1,
9870 8, 0, 9, 0,
9871 10, 0, 11, 0,
9872 12, 0, 13, 0,
9873 14, 0, 15, 15));
9874 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
9875 2,
9876 16, 0, 17, 0,
9877 18, 0, 19, 0,
9878 20, 0, 21, 0,
9879 22, 0, 23, 0));
9880 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
9881 3,
9882 24, 0, 25, 0,
9883 26, 0, 27, 0,
9884 28, 0, 29, 0,
9885 30, 0, 31, 0));
9886
9887 /* DC maps received packets */
9888 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
9889 15_0,
9890 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
9891 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
9892 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
9893 31_16,
9894 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
9895 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
9896
9897 /* initialize the cached sc2vl values consistently with h/w */
9898 for (i = 0; i < 32; i++) {
9899 if (i < 8 || i == 15)
9900 *((u8 *)(dd->sc2vl) + i) = (u8)i;
9901 else
9902 *((u8 *)(dd->sc2vl) + i) = 0;
9903 }
9904}
9905
9906/*
9907 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
9908 * depend on the chip going through a power-on reset - a driver may be loaded
9909 * and unloaded many times.
9910 *
9911 * Do not write any CSR values to the chip in this routine - there may be
9912 * a reset following the (possible) FLR in this routine.
9913 *
9914 */
9915static void init_chip(struct hfi1_devdata *dd)
9916{
9917 int i;
9918
9919 /*
9920 * Put the HFI CSRs in a known state.
9921 * Combine this with a DC reset.
9922 *
9923 * Stop the device from doing anything while we do a
9924 * reset. We know there are no other active users of
9925 * the device since we are now in charge. Turn off
9926 * off all outbound and inbound traffic and make sure
9927 * the device does not generate any interrupts.
9928 */
9929
9930 /* disable send contexts and SDMA engines */
9931 write_csr(dd, SEND_CTRL, 0);
9932 for (i = 0; i < dd->chip_send_contexts; i++)
9933 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
9934 for (i = 0; i < dd->chip_sdma_engines; i++)
9935 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
9936 /* disable port (turn off RXE inbound traffic) and contexts */
9937 write_csr(dd, RCV_CTRL, 0);
9938 for (i = 0; i < dd->chip_rcv_contexts; i++)
9939 write_csr(dd, RCV_CTXT_CTRL, 0);
9940 /* mask all interrupt sources */
9941 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
9942 write_csr(dd, CCE_INT_MASK + (8*i), 0ull);
9943
9944 /*
9945 * DC Reset: do a full DC reset before the register clear.
9946 * A recommended length of time to hold is one CSR read,
9947 * so reread the CceDcCtrl. Then, hold the DC in reset
9948 * across the clear.
9949 */
9950 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
9951 (void) read_csr(dd, CCE_DC_CTRL);
9952
9953 if (use_flr) {
9954 /*
9955 * A FLR will reset the SPC core and part of the PCIe.
9956 * The parts that need to be restored have already been
9957 * saved.
9958 */
9959 dd_dev_info(dd, "Resetting CSRs with FLR\n");
9960
9961 /* do the FLR, the DC reset will remain */
9962 hfi1_pcie_flr(dd);
9963
9964 /* restore command and BARs */
9965 restore_pci_variables(dd);
9966
995deafa 9967 if (is_ax(dd)) {
77241056
MM		 9968 dd_dev_info(dd, "Resetting CSRs with FLR\n");
9969 hfi1_pcie_flr(dd);
9970 restore_pci_variables(dd);
9971 }
9972
7c03ed85 9973 reset_asic_csrs(dd);
77241056
MM		 9974 } else {
9975 dd_dev_info(dd, "Resetting CSRs with writes\n");
9976 reset_cce_csrs(dd);
9977 reset_txe_csrs(dd);
9978 reset_rxe_csrs(dd);
9979 reset_asic_csrs(dd);
9980 reset_misc_csrs(dd);
9981 }
9982 /* clear the DC reset */
9983 write_csr(dd, CCE_DC_CTRL, 0);
7c03ed85 9984
77241056 9985 /* Set the LED off */
995deafa 9986 if (is_ax(dd))
77241056
MM		 9987 setextled(dd, 0);
9988 /*
9989 * Clear the QSFP reset.
72a67ba2 9990 * An FLR enforces a 0 on all out pins. The driver does not touch
77241056 9991 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
72a67ba2 9992 * anything plugged constantly in reset, if it pays attention
77241056 9993 * to RESET_N.
72a67ba2 9994 * Prime examples of this are optical cables. Set all pins high.
77241056
MM		 9995 * I2CCLK and I2CDAT will change per direction, and INT_N and
9996 * MODPRS_N are input only and their value is ignored.
9997 */
72a67ba2
EH		 9998 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
9999 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
77241056
MM		 10000}
10001
10002static void init_early_variables(struct hfi1_devdata *dd)
10003{
10004 int i;
10005
10006 /* assign link credit variables */
10007 dd->vau = CM_VAU;
10008 dd->link_credits = CM_GLOBAL_CREDITS;
995deafa 10009 if (is_ax(dd))
77241056
MM		 10010 dd->link_credits--;
10011 dd->vcu = cu_to_vcu(hfi1_cu);
10012 /* enough room for 8 MAD packets plus header - 17K */
10013 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
10014 if (dd->vl15_init > dd->link_credits)
10015 dd->vl15_init = dd->link_credits;
10016
10017 write_uninitialized_csrs_and_memories(dd);
10018
10019 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10020 for (i = 0; i < dd->num_pports; i++) {
10021 struct hfi1_pportdata *ppd = &dd->pport[i];
10022
10023 set_partition_keys(ppd);
10024 }
10025 init_sc2vl_tables(dd);
10026}
10027
10028static void init_kdeth_qp(struct hfi1_devdata *dd)
10029{
10030 /* user changed the KDETH_QP */
10031 if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
10032 /* out of range or illegal value */
10033 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
10034 kdeth_qp = 0;
10035 }
10036 if (kdeth_qp == 0) /* not set, or failed range check */
10037 kdeth_qp = DEFAULT_KDETH_QP;
10038
10039 write_csr(dd, SEND_BTH_QP,
10040 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK)
10041 << SEND_BTH_QP_KDETH_QP_SHIFT);
10042
10043 write_csr(dd, RCV_BTH_QP,
10044 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK)
10045 << RCV_BTH_QP_KDETH_QP_SHIFT);
10046}
10047
10048/**
10049 * init_qpmap_table
10050 * @dd - device data
10051 * @first_ctxt - first context
10052 * @last_ctxt - first context
10053 *
10054 * This return sets the qpn mapping table that
10055 * is indexed by qpn[8:1].
10056 *
10057 * The routine will round robin the 256 settings
10058 * from first_ctxt to last_ctxt.
10059 *
10060 * The first/last looks ahead to having specialized
10061 * receive contexts for mgmt and bypass. Normal
10062 * verbs traffic will assumed to be on a range
10063 * of receive contexts.
10064 */
10065static void init_qpmap_table(struct hfi1_devdata *dd,
10066 u32 first_ctxt,
10067 u32 last_ctxt)
10068{
10069 u64 reg = 0;
10070 u64 regno = RCV_QP_MAP_TABLE;
10071 int i;
10072 u64 ctxt = first_ctxt;
10073
10074 for (i = 0; i < 256;) {
77241056
MM		 10075 reg |= ctxt << (8 * (i % 8));
10076 i++;
10077 ctxt++;
10078 if (ctxt > last_ctxt)
10079 ctxt = first_ctxt;
10080 if (i % 8 == 0) {
10081 write_csr(dd, regno, reg);
10082 reg = 0;
10083 regno += 8;
10084 }
10085 }
10086 if (i % 8)
10087 write_csr(dd, regno, reg);
10088
10089 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
10090 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
10091}
10092
10093/**
10094 * init_qos - init RX qos
10095 * @dd - device data
10096 * @first_context
10097 *
10098 * This routine initializes Rule 0 and the
10099 * RSM map table to implement qos.
10100 *
10101 * If all of the limit tests succeed,
10102 * qos is applied based on the array
10103 * interpretation of krcvqs where
10104 * entry 0 is VL0.
10105 *
10106 * The number of vl bits (n) and the number of qpn
10107 * bits (m) are computed to feed both the RSM map table
10108 * and the single rule.
10109 *
10110 */
10111static void init_qos(struct hfi1_devdata *dd, u32 first_ctxt)
10112{
10113 u8 max_by_vl = 0;
10114 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
10115 u64 *rsmmap;
10116 u64 reg;
995deafa 10117 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
77241056
MM		 10118
10119 /* validate */
10120 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
10121 num_vls == 1 ||
10122 krcvqsset <= 1)
10123 goto bail;
10124 for (i = 0; i < min_t(unsigned, num_vls, krcvqsset); i++)
10125 if (krcvqs[i] > max_by_vl)
10126 max_by_vl = krcvqs[i];
10127 if (max_by_vl > 32)
10128 goto bail;
10129 qpns_per_vl = __roundup_pow_of_two(max_by_vl);
10130 /* determine bits vl */
10131 n = ilog2(num_vls);
10132 /* determine bits for qpn */
10133 m = ilog2(qpns_per_vl);
10134 if ((m + n) > 7)
10135 goto bail;
10136 if (num_vls * qpns_per_vl > dd->chip_rcv_contexts)
10137 goto bail;
10138 rsmmap = kmalloc_array(NUM_MAP_REGS, sizeof(u64), GFP_KERNEL);
10139 memset(rsmmap, rxcontext, NUM_MAP_REGS * sizeof(u64));
10140 /* init the local copy of the table */
10141 for (i = 0, ctxt = first_ctxt; i < num_vls; i++) {
10142 unsigned tctxt;
10143
10144 for (qpn = 0, tctxt = ctxt;
10145 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
10146 unsigned idx, regoff, regidx;
10147
10148 /* generate index <= 128 */
10149 idx = (qpn << n) ^ i;
10150 regoff = (idx % 8) * 8;
10151 regidx = idx / 8;
10152 reg = rsmmap[regidx];
10153 /* replace 0xff with context number */
10154 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
10155 << regoff);
10156 reg |= (u64)(tctxt++) << regoff;
10157 rsmmap[regidx] = reg;
10158 if (tctxt == ctxt + krcvqs[i])
10159 tctxt = ctxt;
10160 }
10161 ctxt += krcvqs[i];
10162 }
10163 /* flush cached copies to chip */
10164 for (i = 0; i < NUM_MAP_REGS; i++)
10165 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rsmmap[i]);
10166 /* add rule0 */
10167 write_csr(dd, RCV_RSM_CFG /* + (8 * 0) */,
10168 RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_MASK
10169 << RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_SHIFT |
10170 2ull << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
10171 write_csr(dd, RCV_RSM_SELECT /* + (8 * 0) */,
10172 LRH_BTH_MATCH_OFFSET
10173 << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
10174 LRH_SC_MATCH_OFFSET << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
10175 LRH_SC_SELECT_OFFSET << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
10176 ((u64)n) << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
10177 QPN_SELECT_OFFSET << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
10178 ((u64)m + (u64)n) << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
10179 write_csr(dd, RCV_RSM_MATCH /* + (8 * 0) */,
10180 LRH_BTH_MASK << RCV_RSM_MATCH_MASK1_SHIFT |
10181 LRH_BTH_VALUE << RCV_RSM_MATCH_VALUE1_SHIFT |
10182 LRH_SC_MASK << RCV_RSM_MATCH_MASK2_SHIFT |
10183 LRH_SC_VALUE << RCV_RSM_MATCH_VALUE2_SHIFT);
10184 /* Enable RSM */
10185 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
10186 kfree(rsmmap);
82c2611d
NV		 10187 /* map everything else to first context */
10188 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, MIN_KERNEL_KCTXTS - 1);
77241056
MM		 10189 dd->qos_shift = n + 1;
10190 return;
10191bail:
10192 dd->qos_shift = 1;
82c2611d 10193 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
77241056
MM		 10194}
10195
10196static void init_rxe(struct hfi1_devdata *dd)
10197{
10198 /* enable all receive errors */
10199 write_csr(dd, RCV_ERR_MASK, ~0ull);
10200 /* setup QPN map table - start where VL15 context leaves off */
10201 init_qos(
10202 dd,
10203 dd->n_krcv_queues > MIN_KERNEL_KCTXTS ? MIN_KERNEL_KCTXTS : 0);
10204 /*
10205 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
10206 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
10207 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
10208 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
10209 * Max_PayLoad_Size set to its minimum of 128.
10210 *
10211 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
10212 * (64 bytes). Max_Payload_Size is possibly modified upward in
10213 * tune_pcie_caps() which is called after this routine.
10214 */
10215}
10216
10217static void init_other(struct hfi1_devdata *dd)
10218{
10219 /* enable all CCE errors */
10220 write_csr(dd, CCE_ERR_MASK, ~0ull);
10221 /* enable *some* Misc errors */
10222 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
10223 /* enable all DC errors, except LCB */
10224 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
10225 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
10226}
10227
10228/*
10229 * Fill out the given AU table using the given CU. A CU is defined in terms
10230 * AUs. The table is a an encoding: given the index, how many AUs does that
10231 * represent?
10232 *
10233 * NOTE: Assumes that the register layout is the same for the
10234 * local and remote tables.
10235 */
10236static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
10237 u32 csr0to3, u32 csr4to7)
10238{
10239 write_csr(dd, csr0to3,
10240 0ull <<
10241 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT
10242 | 1ull <<
10243 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT
10244 | 2ull * cu <<
10245 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT
10246 | 4ull * cu <<
10247 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
10248 write_csr(dd, csr4to7,
10249 8ull * cu <<
10250 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT
10251 | 16ull * cu <<
10252 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT
10253 | 32ull * cu <<
10254 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT
10255 | 64ull * cu <<
10256 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
10257
10258}
10259
10260static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
10261{
10262 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
10263 SEND_CM_LOCAL_AU_TABLE4_TO7);
10264}
10265
10266void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
10267{
10268 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
10269 SEND_CM_REMOTE_AU_TABLE4_TO7);
10270}
10271
10272static void init_txe(struct hfi1_devdata *dd)
10273{
10274 int i;
10275
10276 /* enable all PIO, SDMA, general, and Egress errors */
10277 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
10278 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
10279 write_csr(dd, SEND_ERR_MASK, ~0ull);
10280 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
10281
10282 /* enable all per-context and per-SDMA engine errors */
10283 for (i = 0; i < dd->chip_send_contexts; i++)
10284 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
10285 for (i = 0; i < dd->chip_sdma_engines; i++)
10286 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
10287
10288 /* set the local CU to AU mapping */
10289 assign_local_cm_au_table(dd, dd->vcu);
10290
10291 /*
10292 * Set reasonable default for Credit Return Timer
10293 * Don't set on Simulator - causes it to choke.
10294 */
10295 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
10296 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
10297}
10298
10299int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
10300{
10301 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
10302 unsigned sctxt;
10303 int ret = 0;
10304 u64 reg;
10305
10306 if (!rcd || !rcd->sc) {
10307 ret = -EINVAL;
10308 goto done;
10309 }
10310 sctxt = rcd->sc->hw_context;
10311 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
10312 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
10313 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
10314 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
10315 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
10316 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
10317 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
10318 /*
10319 * Enable send-side J_KEY integrity check, unless this is A0 h/w
10320 * (due to A0 erratum).
10321 */
995deafa 10322 if (!is_ax(dd)) {
77241056
MM		 10323 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10324 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
10325 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10326 }
10327
10328 /* Enable J_KEY check on receive context. */
10329 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
10330 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
10331 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
10332 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
10333done:
10334 return ret;
10335}
10336
10337int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
10338{
10339 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
10340 unsigned sctxt;
10341 int ret = 0;
10342 u64 reg;
10343
10344 if (!rcd || !rcd->sc) {
10345 ret = -EINVAL;
10346 goto done;
10347 }
10348 sctxt = rcd->sc->hw_context;
10349 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
10350 /*
10351 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
10352 * This check would not have been enabled for A0 h/w, see
10353 * set_ctxt_jkey().
10354 */
995deafa 10355 if (!is_ax(dd)) {
77241056
MM		 10356 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10357 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
10358 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10359 }
10360 /* Turn off the J_KEY on the receive side */
10361 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
10362done:
10363 return ret;
10364}
10365
10366int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
10367{
10368 struct hfi1_ctxtdata *rcd;
10369 unsigned sctxt;
10370 int ret = 0;
10371 u64 reg;
10372
10373 if (ctxt < dd->num_rcv_contexts)
10374 rcd = dd->rcd[ctxt];
10375 else {
10376 ret = -EINVAL;
10377 goto done;
10378 }
10379 if (!rcd || !rcd->sc) {
10380 ret = -EINVAL;
10381 goto done;
10382 }
10383 sctxt = rcd->sc->hw_context;
10384 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
10385 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
10386 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
10387 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10388 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
10389 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10390done:
10391 return ret;
10392}
10393
10394int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
10395{
10396 struct hfi1_ctxtdata *rcd;
10397 unsigned sctxt;
10398 int ret = 0;
10399 u64 reg;
10400
10401 if (ctxt < dd->num_rcv_contexts)
10402 rcd = dd->rcd[ctxt];
10403 else {
10404 ret = -EINVAL;
10405 goto done;
10406 }
10407 if (!rcd || !rcd->sc) {
10408 ret = -EINVAL;
10409 goto done;
10410 }
10411 sctxt = rcd->sc->hw_context;
10412 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10413 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
10414 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10415 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
10416done:
10417 return ret;
10418}
10419
10420/*
10421 * Start doing the clean up the the chip. Our clean up happens in multiple
10422 * stages and this is just the first.
10423 */
10424void hfi1_start_cleanup(struct hfi1_devdata *dd)
10425{
10426 free_cntrs(dd);
10427 free_rcverr(dd);
10428 clean_up_interrupts(dd);
10429}
10430
10431#define HFI_BASE_GUID(dev) \
10432 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
10433
10434/*
10435 * Certain chip functions need to be initialized only once per asic
10436 * instead of per-device. This function finds the peer device and
10437 * checks whether that chip initialization needs to be done by this
10438 * device.
10439 */
10440static void asic_should_init(struct hfi1_devdata *dd)
10441{
10442 unsigned long flags;
10443 struct hfi1_devdata *tmp, *peer = NULL;
10444
10445 spin_lock_irqsave(&hfi1_devs_lock, flags);
10446 /* Find our peer device */
10447 list_for_each_entry(tmp, &hfi1_dev_list, list) {
10448 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
10449 dd->unit != tmp->unit) {
10450 peer = tmp;
10451 break;
10452 }
10453 }
10454
10455 /*
10456 * "Claim" the ASIC for initialization if it hasn't been
10457 " "claimed" yet.
10458 */
10459 if (!peer || !(peer->flags & HFI1_DO_INIT_ASIC))
10460 dd->flags |= HFI1_DO_INIT_ASIC;
10461 spin_unlock_irqrestore(&hfi1_devs_lock, flags);
10462}
10463
10464/**
7c03ed85 10465 * Allocate and initialize the device structure for the hfi.
77241056
MM		 10466 * @dev: the pci_dev for hfi1_ib device
10467 * @ent: pci_device_id struct for this dev
10468 *
10469 * Also allocates, initializes, and returns the devdata struct for this
10470 * device instance
10471 *
10472 * This is global, and is called directly at init to set up the
10473 * chip-specific function pointers for later use.
10474 */
10475struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
10476 const struct pci_device_id *ent)
10477{
10478 struct hfi1_devdata *dd;
10479 struct hfi1_pportdata *ppd;
10480 u64 reg;
10481 int i, ret;
10482 static const char * const inames[] = { /* implementation names */
10483 "RTL silicon",
10484 "RTL VCS simulation",
10485 "RTL FPGA emulation",
10486 "Functional simulator"
10487 };
10488
10489 dd = hfi1_alloc_devdata(pdev,
10490 NUM_IB_PORTS * sizeof(struct hfi1_pportdata));
10491 if (IS_ERR(dd))
10492 goto bail;
10493 ppd = dd->pport;
10494 for (i = 0; i < dd->num_pports; i++, ppd++) {
10495 int vl;
10496 /* init common fields */
10497 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
10498 /* DC supports 4 link widths */
10499 ppd->link_width_supported =
10500 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
10501 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
10502 ppd->link_width_downgrade_supported =
10503 ppd->link_width_supported;
10504 /* start out enabling only 4X */
10505 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
10506 ppd->link_width_downgrade_enabled =
10507 ppd->link_width_downgrade_supported;
10508 /* link width active is 0 when link is down */
10509 /* link width downgrade active is 0 when link is down */
10510
10511 if (num_vls < HFI1_MIN_VLS_SUPPORTED
10512 || num_vls > HFI1_MAX_VLS_SUPPORTED) {
10513 hfi1_early_err(&pdev->dev,
10514 "Invalid num_vls %u, using %u VLs\n",
10515 num_vls, HFI1_MAX_VLS_SUPPORTED);
10516 num_vls = HFI1_MAX_VLS_SUPPORTED;
10517 }
10518 ppd->vls_supported = num_vls;
10519 ppd->vls_operational = ppd->vls_supported;
10520 /* Set the default MTU. */
10521 for (vl = 0; vl < num_vls; vl++)
10522 dd->vld[vl].mtu = hfi1_max_mtu;
10523 dd->vld[15].mtu = MAX_MAD_PACKET;
10524 /*
10525 * Set the initial values to reasonable default, will be set
10526 * for real when link is up.
10527 */
10528 ppd->lstate = IB_PORT_DOWN;
10529 ppd->overrun_threshold = 0x4;
10530 ppd->phy_error_threshold = 0xf;
10531 ppd->port_crc_mode_enabled = link_crc_mask;
10532 /* initialize supported LTP CRC mode */
10533 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
10534 /* initialize enabled LTP CRC mode */
10535 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
10536 /* start in offline */
10537 ppd->host_link_state = HLS_DN_OFFLINE;
10538 init_vl_arb_caches(ppd);
10539 }
10540
10541 dd->link_default = HLS_DN_POLL;
10542
10543 /*
10544 * Do remaining PCIe setup and save PCIe values in dd.
10545 * Any error printing is already done by the init code.
10546 * On return, we have the chip mapped.
10547 */
10548 ret = hfi1_pcie_ddinit(dd, pdev, ent);
10549 if (ret < 0)
10550 goto bail_free;
10551
10552 /* verify that reads actually work, save revision for reset check */
10553 dd->revision = read_csr(dd, CCE_REVISION);
10554 if (dd->revision == ~(u64)0) {
10555 dd_dev_err(dd, "cannot read chip CSRs\n");
10556 ret = -EINVAL;
10557 goto bail_cleanup;
10558 }
10559 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
10560 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
10561 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
10562 & CCE_REVISION_CHIP_REV_MINOR_MASK;
10563
10564 /* obtain the hardware ID - NOT related to unit, which is a
10565 software enumeration */
10566 reg = read_csr(dd, CCE_REVISION2);
10567 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
10568 & CCE_REVISION2_HFI_ID_MASK;
10569 /* the variable size will remove unwanted bits */
10570 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
10571 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
10572 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
10573 dd->icode < ARRAY_SIZE(inames) ? inames[dd->icode] : "unknown",
10574 (int)dd->irev);
10575
10576 /* speeds the hardware can support */
10577 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
10578 /* speeds allowed to run at */
10579 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
10580 /* give a reasonable active value, will be set on link up */
10581 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
10582
10583 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
10584 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
10585 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
10586 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
10587 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
10588 /* fix up link widths for emulation _p */
10589 ppd = dd->pport;
10590 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
10591 ppd->link_width_supported =
10592 ppd->link_width_enabled =
10593 ppd->link_width_downgrade_supported =
10594 ppd->link_width_downgrade_enabled =
10595 OPA_LINK_WIDTH_1X;
10596 }
10597 /* insure num_vls isn't larger than number of sdma engines */
10598 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
10599 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
10600 num_vls, HFI1_MAX_VLS_SUPPORTED);
10601 ppd->vls_supported = num_vls = HFI1_MAX_VLS_SUPPORTED;
10602 ppd->vls_operational = ppd->vls_supported;
10603 }
10604
10605 /*
10606 * Convert the ns parameter to the 64 * cclocks used in the CSR.
10607 * Limit the max if larger than the field holds. If timeout is
10608 * non-zero, then the calculated field will be at least 1.
10609 *
10610 * Must be after icode is set up - the cclock rate depends
10611 * on knowing the hardware being used.
10612 */
10613 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
10614 if (dd->rcv_intr_timeout_csr >
10615 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
10616 dd->rcv_intr_timeout_csr =
10617 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
10618 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
10619 dd->rcv_intr_timeout_csr = 1;
10620
7c03ed85
EH		 10621 /* needs to be done before we look for the peer device */
10622 read_guid(dd);
10623
10624 /* should this device init the ASIC block? */
10625 asic_should_init(dd);
10626
77241056
MM		 10627 /* obtain chip sizes, reset chip CSRs */
10628 init_chip(dd);
10629
10630 /* read in the PCIe link speed information */
10631 ret = pcie_speeds(dd);
10632 if (ret)
10633 goto bail_cleanup;
10634
77241056
MM		 10635 /* read in firmware */
10636 ret = hfi1_firmware_init(dd);
10637 if (ret)
10638 goto bail_cleanup;
10639
10640 /*
10641 * In general, the PCIe Gen3 transition must occur after the
10642 * chip has been idled (so it won't initiate any PCIe transactions
10643 * e.g. an interrupt) and before the driver changes any registers
10644 * (the transition will reset the registers).
10645 *
10646 * In particular, place this call after:
10647 * - init_chip() - the chip will not initiate any PCIe transactions
10648 * - pcie_speeds() - reads the current link speed
10649 * - hfi1_firmware_init() - the needed firmware is ready to be
10650 * downloaded
10651 */
10652 ret = do_pcie_gen3_transition(dd);
10653 if (ret)
10654 goto bail_cleanup;
10655
10656 /* start setting dd values and adjusting CSRs */
10657 init_early_variables(dd);
10658
10659 parse_platform_config(dd);
10660
10661 /* add board names as they are defined */
10662 dd->boardname = kmalloc(64, GFP_KERNEL);
10663 if (!dd->boardname)
10664 goto bail_cleanup;
10665 snprintf(dd->boardname, 64, "Board ID 0x%llx",
10666 dd->revision >> CCE_REVISION_BOARD_ID_LOWER_NIBBLE_SHIFT
10667 & CCE_REVISION_BOARD_ID_LOWER_NIBBLE_MASK);
10668
10669 snprintf(dd->boardversion, BOARD_VERS_MAX,
10670 "ChipABI %u.%u, %s, ChipRev %u.%u, SW Compat %llu\n",
10671 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
10672 dd->boardname,
10673 (u32)dd->majrev,
10674 (u32)dd->minrev,
10675 (dd->revision >> CCE_REVISION_SW_SHIFT)
10676 & CCE_REVISION_SW_MASK);
10677
10678 ret = set_up_context_variables(dd);
10679 if (ret)
10680 goto bail_cleanup;
10681
10682 /* set initial RXE CSRs */
10683 init_rxe(dd);
10684 /* set initial TXE CSRs */
10685 init_txe(dd);
10686 /* set initial non-RXE, non-TXE CSRs */
10687 init_other(dd);
10688 /* set up KDETH QP prefix in both RX and TX CSRs */
10689 init_kdeth_qp(dd);
10690
10691 /* send contexts must be set up before receive contexts */
10692 ret = init_send_contexts(dd);
10693 if (ret)
10694 goto bail_cleanup;
10695
10696 ret = hfi1_create_ctxts(dd);
10697 if (ret)
10698 goto bail_cleanup;
10699
10700 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
10701 /*
10702 * rcd[0] is guaranteed to be valid by this point. Also, all
10703 * context are using the same value, as per the module parameter.
10704 */
10705 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
10706
10707 ret = init_pervl_scs(dd);
10708 if (ret)
10709 goto bail_cleanup;
10710
10711 /* sdma init */
10712 for (i = 0; i < dd->num_pports; ++i) {
10713 ret = sdma_init(dd, i);
10714 if (ret)
10715 goto bail_cleanup;
10716 }
10717
10718 /* use contexts created by hfi1_create_ctxts */
10719 ret = set_up_interrupts(dd);
10720 if (ret)
10721 goto bail_cleanup;
10722
10723 /* set up LCB access - must be after set_up_interrupts() */
10724 init_lcb_access(dd);
10725
10726 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
10727 dd->base_guid & 0xFFFFFF);
10728
10729 dd->oui1 = dd->base_guid >> 56 & 0xFF;
10730 dd->oui2 = dd->base_guid >> 48 & 0xFF;
10731 dd->oui3 = dd->base_guid >> 40 & 0xFF;
10732
10733 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
10734 if (ret)
10735 goto bail_clear_intr;
10736 check_fabric_firmware_versions(dd);
10737
10738 thermal_init(dd);
10739
10740 ret = init_cntrs(dd);
10741 if (ret)
10742 goto bail_clear_intr;
10743
10744 ret = init_rcverr(dd);
10745 if (ret)
10746 goto bail_free_cntrs;
10747
10748 ret = eprom_init(dd);
10749 if (ret)
10750 goto bail_free_rcverr;
10751
10752 goto bail;
10753
10754bail_free_rcverr:
10755 free_rcverr(dd);
10756bail_free_cntrs:
10757 free_cntrs(dd);
10758bail_clear_intr:
10759 clean_up_interrupts(dd);
10760bail_cleanup:
10761 hfi1_pcie_ddcleanup(dd);
10762bail_free:
10763 hfi1_free_devdata(dd);
10764 dd = ERR_PTR(ret);
10765bail:
10766 return dd;
10767}
10768
10769static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
10770 u32 dw_len)
10771{
10772 u32 delta_cycles;
10773 u32 current_egress_rate = ppd->current_egress_rate;
10774 /* rates here are in units of 10^6 bits/sec */
10775
10776 if (desired_egress_rate == -1)
10777 return 0; /* shouldn't happen */
10778
10779 if (desired_egress_rate >= current_egress_rate)
10780 return 0; /* we can't help go faster, only slower */
10781
10782 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
10783 egress_cycles(dw_len * 4, current_egress_rate);
10784
10785 return (u16)delta_cycles;
10786}
10787
10788
10789/**
10790 * create_pbc - build a pbc for transmission
10791 * @flags: special case flags or-ed in built pbc
10792 * @srate: static rate
10793 * @vl: vl
10794 * @dwlen: dword length (header words + data words + pbc words)
10795 *
10796 * Create a PBC with the given flags, rate, VL, and length.
10797 *
10798 * NOTE: The PBC created will not insert any HCRC - all callers but one are
10799 * for verbs, which does not use this PSM feature. The lone other caller
10800 * is for the diagnostic interface which calls this if the user does not
10801 * supply their own PBC.
10802 */
10803u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
10804 u32 dw_len)
10805{
10806 u64 pbc, delay = 0;
10807
10808 if (unlikely(srate_mbs))
10809 delay = delay_cycles(ppd, srate_mbs, dw_len);
10810
10811 pbc = flags
10812 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
10813 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
10814 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
10815 | (dw_len & PBC_LENGTH_DWS_MASK)
10816 << PBC_LENGTH_DWS_SHIFT;
10817
10818 return pbc;
10819}
10820
10821#define SBUS_THERMAL 0x4f
10822#define SBUS_THERM_MONITOR_MODE 0x1
10823
10824#define THERM_FAILURE(dev, ret, reason) \
10825 dd_dev_err((dd), \
10826 "Thermal sensor initialization failed: %s (%d)\n", \
10827 (reason), (ret))
10828
10829/*
10830 * Initialize the Avago Thermal sensor.
10831 *
10832 * After initialization, enable polling of thermal sensor through
10833 * SBus interface. In order for this to work, the SBus Master
10834 * firmware has to be loaded due to the fact that the HW polling
10835 * logic uses SBus interrupts, which are not supported with
10836 * default firmware. Otherwise, no data will be returned through
10837 * the ASIC_STS_THERM CSR.
10838 */
10839static int thermal_init(struct hfi1_devdata *dd)
10840{
10841 int ret = 0;
10842
10843 if (dd->icode != ICODE_RTL_SILICON ||
10844 !(dd->flags & HFI1_DO_INIT_ASIC))
10845 return ret;
10846
10847 acquire_hw_mutex(dd);
10848 dd_dev_info(dd, "Initializing thermal sensor\n");
4ef98989
JAQ		 10849 /* Disable polling of thermal readings */
10850 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
10851 msleep(100);
77241056
MM		 10852 /* Thermal Sensor Initialization */
10853 /* Step 1: Reset the Thermal SBus Receiver */
10854 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10855 RESET_SBUS_RECEIVER, 0);
10856 if (ret) {
10857 THERM_FAILURE(dd, ret, "Bus Reset");
10858 goto done;
10859 }
10860 /* Step 2: Set Reset bit in Thermal block */
10861 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10862 WRITE_SBUS_RECEIVER, 0x1);
10863 if (ret) {
10864 THERM_FAILURE(dd, ret, "Therm Block Reset");
10865 goto done;
10866 }
10867 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
10868 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
10869 WRITE_SBUS_RECEIVER, 0x32);
10870 if (ret) {
10871 THERM_FAILURE(dd, ret, "Write Clock Div");
10872 goto done;
10873 }
10874 /* Step 4: Select temperature mode */
10875 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
10876 WRITE_SBUS_RECEIVER,
10877 SBUS_THERM_MONITOR_MODE);
10878 if (ret) {
10879 THERM_FAILURE(dd, ret, "Write Mode Sel");
10880 goto done;
10881 }
10882 /* Step 5: De-assert block reset and start conversion */
10883 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10884 WRITE_SBUS_RECEIVER, 0x2);
10885 if (ret) {
10886 THERM_FAILURE(dd, ret, "Write Reset Deassert");
10887 goto done;
10888 }
10889 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
10890 msleep(22);
10891
10892 /* Enable polling of thermal readings */
10893 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
10894done:
10895 release_hw_mutex(dd);
10896 return ret;
10897}
10898
10899static void handle_temp_err(struct hfi1_devdata *dd)
10900{
10901 struct hfi1_pportdata *ppd = &dd->pport[0];
10902 /*
10903 * Thermal Critical Interrupt
10904 * Put the device into forced freeze mode, take link down to
10905 * offline, and put DC into reset.
10906 */
10907 dd_dev_emerg(dd,
10908 "Critical temperature reached! Forcing device into freeze mode!\n");
10909 dd->flags |= HFI1_FORCED_FREEZE;
10910 start_freeze_handling(ppd, FREEZE_SELF|FREEZE_ABORT);
10911 /*
10912 * Shut DC down as much and as quickly as possible.
10913 *
10914 * Step 1: Take the link down to OFFLINE. This will cause the
10915 * 8051 to put the Serdes in reset. However, we don't want to
10916 * go through the entire link state machine since we want to
10917 * shutdown ASAP. Furthermore, this is not a graceful shutdown
10918 * but rather an attempt to save the chip.
10919 * Code below is almost the same as quiet_serdes() but avoids
10920 * all the extra work and the sleeps.
10921 */
10922 ppd->driver_link_ready = 0;
10923 ppd->link_enabled = 0;
10924 set_physical_link_state(dd, PLS_OFFLINE |
10925 (OPA_LINKDOWN_REASON_SMA_DISABLED << 8));
10926 /*
10927 * Step 2: Shutdown LCB and 8051
10928 * After shutdown, do not restore DC_CFG_RESET value.
10929 */
10930 dc_shutdown(dd);
10931}
Linux kernel - Deliverables
This page took 0.509906 seconds and 5 git commands to generate.