projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
staging/rdma/hfi1: Add a credit push on diagpkt allocate fail
[deliverable/linux.git] / drivers / staging / rdma / hfi1 / chip.c
CommitLineData
77241056
MM		 1/*
2 *
3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
5 *
6 * GPL LICENSE SUMMARY
7 *
8 * Copyright(c) 2015 Intel Corporation.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * BSD LICENSE
20 *
21 * Copyright(c) 2015 Intel Corporation.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 *
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
32 * distribution.
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 */
50
51/*
52 * This file contains all of the code that is specific to the HFI chip
53 */
54
55#include <linux/pci.h>
56#include <linux/delay.h>
57#include <linux/interrupt.h>
58#include <linux/module.h>
59
60#include "hfi.h"
61#include "trace.h"
62#include "mad.h"
63#include "pio.h"
64#include "sdma.h"
65#include "eprom.h"
5d9157aa 66#include "efivar.h"
77241056
MM		 67
68#define NUM_IB_PORTS 1
69
70uint kdeth_qp;
71module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
72MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
73
74uint num_vls = HFI1_MAX_VLS_SUPPORTED;
75module_param(num_vls, uint, S_IRUGO);
76MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
77
78/*
79 * Default time to aggregate two 10K packets from the idle state
80 * (timer not running). The timer starts at the end of the first packet,
81 * so only the time for one 10K packet and header plus a bit extra is needed.
82 * 10 * 1024 + 64 header byte = 10304 byte
83 * 10304 byte / 12.5 GB/s = 824.32ns
84 */
85uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
86module_param(rcv_intr_timeout, uint, S_IRUGO);
87MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
88
89uint rcv_intr_count = 16; /* same as qib */
90module_param(rcv_intr_count, uint, S_IRUGO);
91MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
92
93ushort link_crc_mask = SUPPORTED_CRCS;
94module_param(link_crc_mask, ushort, S_IRUGO);
95MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
96
97uint loopback;
98module_param_named(loopback, loopback, uint, S_IRUGO);
99MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
100
101/* Other driver tunables */
102uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
103static ushort crc_14b_sideband = 1;
104static uint use_flr = 1;
105uint quick_linkup; /* skip LNI */
106
107struct flag_table {
108 u64 flag; /* the flag */
109 char *str; /* description string */
110 u16 extra; /* extra information */
111 u16 unused0;
112 u32 unused1;
113};
114
115/* str must be a string constant */
116#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
117#define FLAG_ENTRY0(str, flag) {flag, str, 0}
118
119/* Send Error Consequences */
120#define SEC_WRITE_DROPPED 0x1
121#define SEC_PACKET_DROPPED 0x2
122#define SEC_SC_HALTED 0x4 /* per-context only */
123#define SEC_SPC_FREEZE 0x8 /* per-HFI only */
124
77241056 125#define MIN_KERNEL_KCTXTS 2
82c2611d 126#define FIRST_KERNEL_KCTXT 1
77241056
MM		 127#define NUM_MAP_REGS 32
128
129/* Bit offset into the GUID which carries HFI id information */
130#define GUID_HFI_INDEX_SHIFT 39
131
132/* extract the emulation revision */
133#define emulator_rev(dd) ((dd)->irev >> 8)
134/* parallel and serial emulation versions are 3 and 4 respectively */
135#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
136#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
137
138/* RSM fields */
139
140/* packet type */
141#define IB_PACKET_TYPE 2ull
142#define QW_SHIFT 6ull
143/* QPN[7..1] */
144#define QPN_WIDTH 7ull
145
146/* LRH.BTH: QW 0, OFFSET 48 - for match */
147#define LRH_BTH_QW 0ull
148#define LRH_BTH_BIT_OFFSET 48ull
149#define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
150#define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
151#define LRH_BTH_SELECT
152#define LRH_BTH_MASK 3ull
153#define LRH_BTH_VALUE 2ull
154
155/* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
156#define LRH_SC_QW 0ull
157#define LRH_SC_BIT_OFFSET 56ull
158#define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
159#define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
160#define LRH_SC_MASK 128ull
161#define LRH_SC_VALUE 0ull
162
163/* SC[n..0] QW 0, OFFSET 60 - for select */
164#define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
165
166/* QPN[m+n:1] QW 1, OFFSET 1 */
167#define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
168
169/* defines to build power on SC2VL table */
170#define SC2VL_VAL( \
171 num, \
172 sc0, sc0val, \
173 sc1, sc1val, \
174 sc2, sc2val, \
175 sc3, sc3val, \
176 sc4, sc4val, \
177 sc5, sc5val, \
178 sc6, sc6val, \
179 sc7, sc7val) \
180( \
181 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
182 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
183 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
184 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
185 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
186 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
187 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
188 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
189)
190
191#define DC_SC_VL_VAL( \
192 range, \
193 e0, e0val, \
194 e1, e1val, \
195 e2, e2val, \
196 e3, e3val, \
197 e4, e4val, \
198 e5, e5val, \
199 e6, e6val, \
200 e7, e7val, \
201 e8, e8val, \
202 e9, e9val, \
203 e10, e10val, \
204 e11, e11val, \
205 e12, e12val, \
206 e13, e13val, \
207 e14, e14val, \
208 e15, e15val) \
209( \
210 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
211 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
212 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
213 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
214 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
215 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
216 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
217 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
218 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
219 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
220 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
221 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
222 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
223 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
224 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
225 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
226)
227
228/* all CceStatus sub-block freeze bits */
229#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
230 | CCE_STATUS_RXE_FROZE_SMASK \
231 | CCE_STATUS_TXE_FROZE_SMASK \
232 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
233/* all CceStatus sub-block TXE pause bits */
234#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
235 | CCE_STATUS_TXE_PAUSED_SMASK \
236 | CCE_STATUS_SDMA_PAUSED_SMASK)
237/* all CceStatus sub-block RXE pause bits */
238#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
239
240/*
241 * CCE Error flags.
242 */
243static struct flag_table cce_err_status_flags[] = {
244/* 0*/ FLAG_ENTRY0("CceCsrParityErr",
245 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
246/* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
247 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
248/* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
249 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
250/* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
251 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
252/* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
253 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
254/* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
255 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
256/* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
257 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
258/* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
259 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
260/* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
261 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
262/* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
263 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
264/*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
265 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
266/*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
267 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
268/*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
269 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
270/*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
271 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
272/*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
273 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
274/*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
275 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
276/*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
277 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
278/*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
279 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
280/*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
281 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
282/*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
283 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
284/*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
285 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
286/*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
287 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
288/*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
289 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
290/*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
291 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
292/*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
293 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
294/*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
295 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
296/*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
297 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
298/*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
299 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
300/*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
301 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
302/*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
303 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
304/*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
305 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
306/*31*/ FLAG_ENTRY0("LATriggered",
307 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
308/*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
309 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
310/*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
311 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
312/*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
313 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
314/*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
315 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
316/*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
317 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
318/*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
319 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
320/*38*/ FLAG_ENTRY0("CceIntMapCorErr",
321 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
322/*39*/ FLAG_ENTRY0("CceIntMapUncErr",
323 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
324/*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
325 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
326/*41-63 reserved*/
327};
328
329/*
330 * Misc Error flags
331 */
332#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
333static struct flag_table misc_err_status_flags[] = {
334/* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
335/* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
336/* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
337/* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
338/* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
339/* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
340/* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
341/* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
342/* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
343/* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
344/*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
345/*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
346/*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
347};
348
349/*
350 * TXE PIO Error flags and consequences
351 */
352static struct flag_table pio_err_status_flags[] = {
353/* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
354 SEC_WRITE_DROPPED,
355 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
356/* 1*/ FLAG_ENTRY("PioWriteAddrParity",
357 SEC_SPC_FREEZE,
358 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
359/* 2*/ FLAG_ENTRY("PioCsrParity",
360 SEC_SPC_FREEZE,
361 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
362/* 3*/ FLAG_ENTRY("PioSbMemFifo0",
363 SEC_SPC_FREEZE,
364 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
365/* 4*/ FLAG_ENTRY("PioSbMemFifo1",
366 SEC_SPC_FREEZE,
367 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
368/* 5*/ FLAG_ENTRY("PioPccFifoParity",
369 SEC_SPC_FREEZE,
370 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
371/* 6*/ FLAG_ENTRY("PioPecFifoParity",
372 SEC_SPC_FREEZE,
373 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
374/* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
375 SEC_SPC_FREEZE,
376 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
377/* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
378 SEC_SPC_FREEZE,
379 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
380/* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
381 SEC_SPC_FREEZE,
382 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
383/*10*/ FLAG_ENTRY("PioSmPktResetParity",
384 SEC_SPC_FREEZE,
385 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
386/*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
387 SEC_SPC_FREEZE,
388 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
389/*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
390 SEC_SPC_FREEZE,
391 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
392/*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
393 0,
394 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
395/*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
396 0,
397 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
398/*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
399 SEC_SPC_FREEZE,
400 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
401/*16*/ FLAG_ENTRY("PioPpmcPblFifo",
402 SEC_SPC_FREEZE,
403 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
404/*17*/ FLAG_ENTRY("PioInitSmIn",
405 0,
406 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
407/*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
408 SEC_SPC_FREEZE,
409 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
410/*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
411 SEC_SPC_FREEZE,
412 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
413/*20*/ FLAG_ENTRY("PioHostAddrMemCor",
414 0,
415 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
416/*21*/ FLAG_ENTRY("PioWriteDataParity",
417 SEC_SPC_FREEZE,
418 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
419/*22*/ FLAG_ENTRY("PioStateMachine",
420 SEC_SPC_FREEZE,
421 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
422/*23*/ FLAG_ENTRY("PioWriteQwValidParity",
423 SEC_WRITE_DROPPED|SEC_SPC_FREEZE,
424 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
425/*24*/ FLAG_ENTRY("PioBlockQwCountParity",
426 SEC_WRITE_DROPPED|SEC_SPC_FREEZE,
427 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
428/*25*/ FLAG_ENTRY("PioVlfVlLenParity",
429 SEC_SPC_FREEZE,
430 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
431/*26*/ FLAG_ENTRY("PioVlfSopParity",
432 SEC_SPC_FREEZE,
433 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
434/*27*/ FLAG_ENTRY("PioVlFifoParity",
435 SEC_SPC_FREEZE,
436 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
437/*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
438 SEC_SPC_FREEZE,
439 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
440/*29*/ FLAG_ENTRY("PioPpmcSopLen",
441 SEC_SPC_FREEZE,
442 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
443/*30-31 reserved*/
444/*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
445 SEC_SPC_FREEZE,
446 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
447/*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
448 SEC_SPC_FREEZE,
449 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
450/*34*/ FLAG_ENTRY("PioPccSopHeadParity",
451 SEC_SPC_FREEZE,
452 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
453/*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
454 SEC_SPC_FREEZE,
455 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
456/*36-63 reserved*/
457};
458
459/* TXE PIO errors that cause an SPC freeze */
460#define ALL_PIO_FREEZE_ERR \
461 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
462 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
463 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
464 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
465 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
466 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
467 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
468 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
469 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
470 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
471 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
489 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
490
491/*
492 * TXE SDMA Error flags
493 */
494static struct flag_table sdma_err_status_flags[] = {
495/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
496 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
497/* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
498 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
499/* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
500 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
501/* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
502 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
503/*04-63 reserved*/
504};
505
506/* TXE SDMA errors that cause an SPC freeze */
507#define ALL_SDMA_FREEZE_ERR \
508 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
509 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
510 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
511
512/*
513 * TXE Egress Error flags
514 */
515#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
516static struct flag_table egress_err_status_flags[] = {
517/* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
518/* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
519/* 2 reserved */
520/* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
521 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
522/* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
523/* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
524/* 6 reserved */
525/* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
526 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
527/* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
528 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
529/* 9-10 reserved */
530/*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
531 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
532/*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
533/*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
534/*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
535/*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
536/*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
537 SEES(TX_SDMA0_DISALLOWED_PACKET)),
538/*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
539 SEES(TX_SDMA1_DISALLOWED_PACKET)),
540/*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
541 SEES(TX_SDMA2_DISALLOWED_PACKET)),
542/*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
543 SEES(TX_SDMA3_DISALLOWED_PACKET)),
544/*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
545 SEES(TX_SDMA4_DISALLOWED_PACKET)),
546/*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
547 SEES(TX_SDMA5_DISALLOWED_PACKET)),
548/*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
549 SEES(TX_SDMA6_DISALLOWED_PACKET)),
550/*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
551 SEES(TX_SDMA7_DISALLOWED_PACKET)),
552/*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
553 SEES(TX_SDMA8_DISALLOWED_PACKET)),
554/*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
555 SEES(TX_SDMA9_DISALLOWED_PACKET)),
556/*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
557 SEES(TX_SDMA10_DISALLOWED_PACKET)),
558/*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
559 SEES(TX_SDMA11_DISALLOWED_PACKET)),
560/*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
561 SEES(TX_SDMA12_DISALLOWED_PACKET)),
562/*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
563 SEES(TX_SDMA13_DISALLOWED_PACKET)),
564/*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
565 SEES(TX_SDMA14_DISALLOWED_PACKET)),
566/*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
567 SEES(TX_SDMA15_DISALLOWED_PACKET)),
568/*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
569 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
570/*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
571 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
572/*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
573 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
574/*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
575 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
576/*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
577 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
578/*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
579 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
580/*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
581 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
582/*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
583 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
584/*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
585 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
586/*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
587/*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
588/*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
589/*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
590/*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
591/*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
592/*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
593/*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
594/*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
595/*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
596/*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
597/*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
598/*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
599/*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
600/*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
601/*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
602/*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
603/*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
604/*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
605/*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
606/*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
607/*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
608 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
609/*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
610 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
611};
612
613/*
614 * TXE Egress Error Info flags
615 */
616#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
617static struct flag_table egress_err_info_flags[] = {
618/* 0*/ FLAG_ENTRY0("Reserved", 0ull),
619/* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
620/* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
621/* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
622/* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
623/* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
624/* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
625/* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
626/* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
627/* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
628/*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
629/*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
630/*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
631/*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
632/*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
633/*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
634/*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
635/*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
636/*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
637/*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
638/*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
639/*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
640};
641
642/* TXE Egress errors that cause an SPC freeze */
643#define ALL_TXE_EGRESS_FREEZE_ERR \
644 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
645 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
646 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
647 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
648 | SEES(TX_LAUNCH_CSR_PARITY) \
649 | SEES(TX_SBRD_CTL_CSR_PARITY) \
650 | SEES(TX_CONFIG_PARITY) \
651 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
652 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
653 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
654 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
655 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
656 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
657 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
658 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
659 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
660 | SEES(TX_CREDIT_RETURN_PARITY))
661
662/*
663 * TXE Send error flags
664 */
665#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
666static struct flag_table send_err_status_flags[] = {
667/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", SES(CSR_PARITY)),
668/* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
669/* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
670};
671
672/*
673 * TXE Send Context Error flags and consequences
674 */
675static struct flag_table sc_err_status_flags[] = {
676/* 0*/ FLAG_ENTRY("InconsistentSop",
677 SEC_PACKET_DROPPED | SEC_SC_HALTED,
678 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
679/* 1*/ FLAG_ENTRY("DisallowedPacket",
680 SEC_PACKET_DROPPED | SEC_SC_HALTED,
681 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
682/* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
683 SEC_WRITE_DROPPED | SEC_SC_HALTED,
684 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
685/* 3*/ FLAG_ENTRY("WriteOverflow",
686 SEC_WRITE_DROPPED | SEC_SC_HALTED,
687 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
688/* 4*/ FLAG_ENTRY("WriteOutOfBounds",
689 SEC_WRITE_DROPPED | SEC_SC_HALTED,
690 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
691/* 5-63 reserved*/
692};
693
694/*
695 * RXE Receive Error flags
696 */
697#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
698static struct flag_table rxe_err_status_flags[] = {
699/* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
700/* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
701/* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
702/* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
703/* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
704/* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
705/* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
706/* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
707/* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
708/* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
709/*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
710/*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
711/*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
712/*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
713/*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
714/*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
715/*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
716 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
717/*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
718/*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
719/*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
720 RXES(RBUF_BLOCK_LIST_READ_UNC)),
721/*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
722 RXES(RBUF_BLOCK_LIST_READ_COR)),
723/*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
724 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
725/*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
726 RXES(RBUF_CSR_QENT_CNT_PARITY)),
727/*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
728 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
729/*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
730 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
731/*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
732/*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
733/*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
734 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
735/*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
736/*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
737/*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
738/*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
739/*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
740/*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
741/*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
742/*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
743 RXES(RBUF_FL_INITDONE_PARITY)),
744/*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
745 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
746/*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
747/*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
748/*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
749/*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
750 RXES(LOOKUP_DES_PART1_UNC_COR)),
751/*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
752 RXES(LOOKUP_DES_PART2_PARITY)),
753/*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
754/*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
755/*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
756/*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
757/*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
758/*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
759/*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
760/*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
761/*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
762/*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
763/*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
764/*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
765/*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
766/*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
767/*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
768/*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
769/*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
770/*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
771/*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
772/*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
773/*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
774/*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
775};
776
777/* RXE errors that will trigger an SPC freeze */
778#define ALL_RXE_FREEZE_ERR \
779 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
780 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
781 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
782 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
783 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
784 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
785 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
786 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
787 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
788 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
789 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
790 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
791 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
792 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
793 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
794 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
795 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
823
824#define RXE_FREEZE_ABORT_MASK \
825 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
826 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
827 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
828
829/*
830 * DCC Error Flags
831 */
832#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
833static struct flag_table dcc_err_flags[] = {
834 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
835 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
836 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
837 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
838 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
839 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
840 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
841 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
842 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
843 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
844 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
845 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
846 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
847 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
848 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
849 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
850 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
851 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
852 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
853 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
854 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
855 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
856 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
857 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
858 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
859 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
860 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
861 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
862 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
863 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
864 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
865 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
866 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
867 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
868 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
869 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
870 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
871 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
872 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
873 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
874 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
875 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
876 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
877 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
878 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
879 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
880};
881
882/*
883 * LCB error flags
884 */
885#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
886static struct flag_table lcb_err_flags[] = {
887/* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
888/* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
889/* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
890/* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
891 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
892/* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
893/* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
894/* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
895/* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
896/* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
897/* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
898/*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
899/*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
900/*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
901/*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
902 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
903/*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
904/*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
905/*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
906/*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
907/*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
908/*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
909 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
910/*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
911/*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
912/*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
913/*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
914/*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
915/*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
916/*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
917 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
918/*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
919/*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
920 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
921/*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
922 LCBE(REDUNDANT_FLIT_PARITY_ERR))
923};
924
925/*
926 * DC8051 Error Flags
927 */
928#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
929static struct flag_table dc8051_err_flags[] = {
930 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
931 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
932 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
933 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
934 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
935 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
936 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
937 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
938 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
939 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
940 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
941};
942
943/*
944 * DC8051 Information Error flags
945 *
946 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
947 */
948static struct flag_table dc8051_info_err_flags[] = {
949 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
950 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
951 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
952 FLAG_ENTRY0("Serdes internal loopback failure",
953 FAILED_SERDES_INTERNAL_LOOPBACK),
954 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
955 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
956 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
957 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
958 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
959 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
960 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
961 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT)
962};
963
964/*
965 * DC8051 Information Host Information flags
966 *
967 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
968 */
969static struct flag_table dc8051_info_host_msg_flags[] = {
970 FLAG_ENTRY0("Host request done", 0x0001),
971 FLAG_ENTRY0("BC SMA message", 0x0002),
972 FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
973 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
974 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
975 FLAG_ENTRY0("External device config request", 0x0020),
976 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
977 FLAG_ENTRY0("LinkUp achieved", 0x0080),
978 FLAG_ENTRY0("Link going down", 0x0100),
979};
980
981
982static u32 encoded_size(u32 size);
983static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
984static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
985static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
986 u8 *continuous);
987static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
988 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
989static void read_vc_remote_link_width(struct hfi1_devdata *dd,
990 u8 *remote_tx_rate, u16 *link_widths);
991static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
992 u8 *flag_bits, u16 *link_widths);
993static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
994 u8 *device_rev);
995static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
996static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
997static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
998 u8 *tx_polarity_inversion,
999 u8 *rx_polarity_inversion, u8 *max_rate);
1000static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1001 unsigned int context, u64 err_status);
1002static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1003static void handle_dcc_err(struct hfi1_devdata *dd,
1004 unsigned int context, u64 err_status);
1005static void handle_lcb_err(struct hfi1_devdata *dd,
1006 unsigned int context, u64 err_status);
1007static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1008static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1009static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1010static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1011static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1012static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1013static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1014static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1015static void set_partition_keys(struct hfi1_pportdata *);
1016static const char *link_state_name(u32 state);
1017static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1018 u32 state);
1019static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1020 u64 *out_data);
1021static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1022static int thermal_init(struct hfi1_devdata *dd);
1023
1024static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1025 int msecs);
1026static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1027static void handle_temp_err(struct hfi1_devdata *);
1028static void dc_shutdown(struct hfi1_devdata *);
1029static void dc_start(struct hfi1_devdata *);
1030
1031/*
1032 * Error interrupt table entry. This is used as input to the interrupt
1033 * "clear down" routine used for all second tier error interrupt register.
1034 * Second tier interrupt registers have a single bit representing them
1035 * in the top-level CceIntStatus.
1036 */
1037struct err_reg_info {
1038 u32 status; /* status CSR offset */
1039 u32 clear; /* clear CSR offset */
1040 u32 mask; /* mask CSR offset */
1041 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1042 const char *desc;
1043};
1044
1045#define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1046#define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1047#define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1048
1049/*
1050 * Helpers for building HFI and DC error interrupt table entries. Different
1051 * helpers are needed because of inconsistent register names.
1052 */
1053#define EE(reg, handler, desc) \
1054 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1055 handler, desc }
1056#define DC_EE1(reg, handler, desc) \
1057 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1058#define DC_EE2(reg, handler, desc) \
1059 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1060
1061/*
1062 * Table of the "misc" grouping of error interrupts. Each entry refers to
1063 * another register containing more information.
1064 */
1065static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1066/* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1067/* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1068/* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1069/* 3*/ { 0, 0, 0, NULL }, /* reserved */
1070/* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1071/* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1072/* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1073/* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1074 /* the rest are reserved */
1075};
1076
1077/*
1078 * Index into the Various section of the interrupt sources
1079 * corresponding to the Critical Temperature interrupt.
1080 */
1081#define TCRIT_INT_SOURCE 4
1082
1083/*
1084 * SDMA error interrupt entry - refers to another register containing more
1085 * information.
1086 */
1087static const struct err_reg_info sdma_eng_err =
1088 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1089
1090static const struct err_reg_info various_err[NUM_VARIOUS] = {
1091/* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1092/* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1093/* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1094/* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1095/* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1096 /* rest are reserved */
1097};
1098
1099/*
1100 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1101 * register can not be derived from the MTU value because 10K is not
1102 * a power of 2. Therefore, we need a constant. Everything else can
1103 * be calculated.
1104 */
1105#define DCC_CFG_PORT_MTU_CAP_10240 7
1106
1107/*
1108 * Table of the DC grouping of error interrupts. Each entry refers to
1109 * another register containing more information.
1110 */
1111static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1112/* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1113/* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1114/* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1115/* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1116 /* the rest are reserved */
1117};
1118
1119struct cntr_entry {
1120 /*
1121 * counter name
1122 */
1123 char *name;
1124
1125 /*
1126 * csr to read for name (if applicable)
1127 */
1128 u64 csr;
1129
1130 /*
1131 * offset into dd or ppd to store the counter's value
1132 */
1133 int offset;
1134
1135 /*
1136 * flags
1137 */
1138 u8 flags;
1139
1140 /*
1141 * accessor for stat element, context either dd or ppd
1142 */
1143 u64 (*rw_cntr)(const struct cntr_entry *,
1144 void *context,
1145 int vl,
1146 int mode,
1147 u64 data);
1148};
1149
1150#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1151#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1152
1153#define CNTR_ELEM(name, csr, offset, flags, accessor) \
1154{ \
1155 name, \
1156 csr, \
1157 offset, \
1158 flags, \
1159 accessor \
1160}
1161
1162/* 32bit RXE */
1163#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1164CNTR_ELEM(#name, \
1165 (counter * 8 + RCV_COUNTER_ARRAY32), \
1166 0, flags | CNTR_32BIT, \
1167 port_access_u32_csr)
1168
1169#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1170CNTR_ELEM(#name, \
1171 (counter * 8 + RCV_COUNTER_ARRAY32), \
1172 0, flags | CNTR_32BIT, \
1173 dev_access_u32_csr)
1174
1175/* 64bit RXE */
1176#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1177CNTR_ELEM(#name, \
1178 (counter * 8 + RCV_COUNTER_ARRAY64), \
1179 0, flags, \
1180 port_access_u64_csr)
1181
1182#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1183CNTR_ELEM(#name, \
1184 (counter * 8 + RCV_COUNTER_ARRAY64), \
1185 0, flags, \
1186 dev_access_u64_csr)
1187
1188#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1189#define OVR_ELM(ctx) \
1190CNTR_ELEM("RcvHdrOvr" #ctx, \
1191 (RCV_HDR_OVFL_CNT + ctx*0x100), \
1192 0, CNTR_NORMAL, port_access_u64_csr)
1193
1194/* 32bit TXE */
1195#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1196CNTR_ELEM(#name, \
1197 (counter * 8 + SEND_COUNTER_ARRAY32), \
1198 0, flags | CNTR_32BIT, \
1199 port_access_u32_csr)
1200
1201/* 64bit TXE */
1202#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1203CNTR_ELEM(#name, \
1204 (counter * 8 + SEND_COUNTER_ARRAY64), \
1205 0, flags, \
1206 port_access_u64_csr)
1207
1208# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1209CNTR_ELEM(#name,\
1210 counter * 8 + SEND_COUNTER_ARRAY64, \
1211 0, \
1212 flags, \
1213 dev_access_u64_csr)
1214
1215/* CCE */
1216#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1217CNTR_ELEM(#name, \
1218 (counter * 8 + CCE_COUNTER_ARRAY32), \
1219 0, flags | CNTR_32BIT, \
1220 dev_access_u32_csr)
1221
1222#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1223CNTR_ELEM(#name, \
1224 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1225 0, flags | CNTR_32BIT, \
1226 dev_access_u32_csr)
1227
1228/* DC */
1229#define DC_PERF_CNTR(name, counter, flags) \
1230CNTR_ELEM(#name, \
1231 counter, \
1232 0, \
1233 flags, \
1234 dev_access_u64_csr)
1235
1236#define DC_PERF_CNTR_LCB(name, counter, flags) \
1237CNTR_ELEM(#name, \
1238 counter, \
1239 0, \
1240 flags, \
1241 dc_access_lcb_cntr)
1242
1243/* ibp counters */
1244#define SW_IBP_CNTR(name, cntr) \
1245CNTR_ELEM(#name, \
1246 0, \
1247 0, \
1248 CNTR_SYNTH, \
1249 access_ibp_##cntr)
1250
1251u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1252{
1253 u64 val;
1254
1255 if (dd->flags & HFI1_PRESENT) {
1256 val = readq((void __iomem *)dd->kregbase + offset);
1257 return val;
1258 }
1259 return -1;
1260}
1261
1262void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1263{
1264 if (dd->flags & HFI1_PRESENT)
1265 writeq(value, (void __iomem *)dd->kregbase + offset);
1266}
1267
1268void __iomem *get_csr_addr(
1269 struct hfi1_devdata *dd,
1270 u32 offset)
1271{
1272 return (void __iomem *)dd->kregbase + offset;
1273}
1274
1275static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1276 int mode, u64 value)
1277{
1278 u64 ret;
1279
1280
1281 if (mode == CNTR_MODE_R) {
1282 ret = read_csr(dd, csr);
1283 } else if (mode == CNTR_MODE_W) {
1284 write_csr(dd, csr, value);
1285 ret = value;
1286 } else {
1287 dd_dev_err(dd, "Invalid cntr register access mode");
1288 return 0;
1289 }
1290
1291 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1292 return ret;
1293}
1294
1295/* Dev Access */
1296static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1297 void *context, int vl, int mode, u64 data)
1298{
a787bde8 1299 struct hfi1_devdata *dd = context;
77241056
MM		 1300
1301 if (vl != CNTR_INVALID_VL)
1302 return 0;
1303 return read_write_csr(dd, entry->csr, mode, data);
1304}
1305
1306static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1307 int vl, int mode, u64 data)
1308{
a787bde8 1309 struct hfi1_devdata *dd = context;
77241056
MM		 1310
1311 u64 val = 0;
1312 u64 csr = entry->csr;
1313
1314 if (entry->flags & CNTR_VL) {
1315 if (vl == CNTR_INVALID_VL)
1316 return 0;
1317 csr += 8 * vl;
1318 } else {
1319 if (vl != CNTR_INVALID_VL)
1320 return 0;
1321 }
1322
1323 val = read_write_csr(dd, csr, mode, data);
1324 return val;
1325}
1326
1327static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1328 int vl, int mode, u64 data)
1329{
a787bde8 1330 struct hfi1_devdata *dd = context;
77241056
MM		 1331 u32 csr = entry->csr;
1332 int ret = 0;
1333
1334 if (vl != CNTR_INVALID_VL)
1335 return 0;
1336 if (mode == CNTR_MODE_R)
1337 ret = read_lcb_csr(dd, csr, &data);
1338 else if (mode == CNTR_MODE_W)
1339 ret = write_lcb_csr(dd, csr, data);
1340
1341 if (ret) {
1342 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1343 return 0;
1344 }
1345
1346 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1347 return data;
1348}
1349
1350/* Port Access */
1351static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1352 int vl, int mode, u64 data)
1353{
a787bde8 1354 struct hfi1_pportdata *ppd = context;
77241056
MM		 1355
1356 if (vl != CNTR_INVALID_VL)
1357 return 0;
1358 return read_write_csr(ppd->dd, entry->csr, mode, data);
1359}
1360
1361static u64 port_access_u64_csr(const struct cntr_entry *entry,
1362 void *context, int vl, int mode, u64 data)
1363{
a787bde8 1364 struct hfi1_pportdata *ppd = context;
77241056
MM		 1365 u64 val;
1366 u64 csr = entry->csr;
1367
1368 if (entry->flags & CNTR_VL) {
1369 if (vl == CNTR_INVALID_VL)
1370 return 0;
1371 csr += 8 * vl;
1372 } else {
1373 if (vl != CNTR_INVALID_VL)
1374 return 0;
1375 }
1376 val = read_write_csr(ppd->dd, csr, mode, data);
1377 return val;
1378}
1379
1380/* Software defined */
1381static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1382 u64 data)
1383{
1384 u64 ret;
1385
1386 if (mode == CNTR_MODE_R) {
1387 ret = *cntr;
1388 } else if (mode == CNTR_MODE_W) {
1389 *cntr = data;
1390 ret = data;
1391 } else {
1392 dd_dev_err(dd, "Invalid cntr sw access mode");
1393 return 0;
1394 }
1395
1396 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1397
1398 return ret;
1399}
1400
1401static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1402 int vl, int mode, u64 data)
1403{
a787bde8 1404 struct hfi1_pportdata *ppd = context;
77241056
MM		 1405
1406 if (vl != CNTR_INVALID_VL)
1407 return 0;
1408 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1409}
1410
1411static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1412 int vl, int mode, u64 data)
1413{
a787bde8 1414 struct hfi1_pportdata *ppd = context;
77241056
MM		 1415
1416 if (vl != CNTR_INVALID_VL)
1417 return 0;
1418 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1419}
1420
1421static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1422 void *context, int vl, int mode, u64 data)
1423{
a787bde8 1424 struct hfi1_pportdata *ppd = context;
77241056
MM		 1425
1426 if (vl != CNTR_INVALID_VL)
1427 return 0;
1428
1429 return read_write_sw(ppd->dd, &ppd->port_xmit_discards, mode, data);
1430}
1431
1432static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1433 void *context, int vl, int mode, u64 data)
1434{
a787bde8 1435 struct hfi1_pportdata *ppd = context;
77241056
MM		 1436
1437 if (vl != CNTR_INVALID_VL)
1438 return 0;
1439
1440 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1441 mode, data);
1442}
1443
1444static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1445 void *context, int vl, int mode, u64 data)
1446{
a787bde8 1447 struct hfi1_pportdata *ppd = context;
77241056
MM		 1448
1449 if (vl != CNTR_INVALID_VL)
1450 return 0;
1451
1452 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1453 mode, data);
1454}
1455
1456u64 get_all_cpu_total(u64 __percpu *cntr)
1457{
1458 int cpu;
1459 u64 counter = 0;
1460
1461 for_each_possible_cpu(cpu)
1462 counter += *per_cpu_ptr(cntr, cpu);
1463 return counter;
1464}
1465
1466static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1467 u64 __percpu *cntr,
1468 int vl, int mode, u64 data)
1469{
1470
1471 u64 ret = 0;
1472
1473 if (vl != CNTR_INVALID_VL)
1474 return 0;
1475
1476 if (mode == CNTR_MODE_R) {
1477 ret = get_all_cpu_total(cntr) - *z_val;
1478 } else if (mode == CNTR_MODE_W) {
1479 /* A write can only zero the counter */
1480 if (data == 0)
1481 *z_val = get_all_cpu_total(cntr);
1482 else
1483 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1484 } else {
1485 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1486 return 0;
1487 }
1488
1489 return ret;
1490}
1491
1492static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1493 void *context, int vl, int mode, u64 data)
1494{
a787bde8 1495 struct hfi1_devdata *dd = context;
77241056
MM		 1496
1497 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1498 mode, data);
1499}
1500
1501static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1502 void *context, int vl, int mode, u64 data)
1503{
a787bde8 1504 struct hfi1_devdata *dd = context;
77241056
MM		 1505
1506 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1507 mode, data);
1508}
1509
1510static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1511 void *context, int vl, int mode, u64 data)
1512{
a787bde8 1513 struct hfi1_devdata *dd = context;
77241056
MM		 1514
1515 return dd->verbs_dev.n_piowait;
1516}
1517
1518static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1519 void *context, int vl, int mode, u64 data)
1520{
a787bde8 1521 struct hfi1_devdata *dd = context;
77241056
MM		 1522
1523 return dd->verbs_dev.n_txwait;
1524}
1525
1526static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1527 void *context, int vl, int mode, u64 data)
1528{
a787bde8 1529 struct hfi1_devdata *dd = context;
77241056
MM		 1530
1531 return dd->verbs_dev.n_kmem_wait;
1532}
1533
b421922e
DL		 1534static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1535 void *context, int vl, int mode, u64 data)
1536{
1537 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1538
1539 return dd->verbs_dev.n_send_schedule;
1540}
1541
77241056
MM		 1542#define def_access_sw_cpu(cntr) \
1543static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
1544 void *context, int vl, int mode, u64 data) \
1545{ \
1546 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
1547 return read_write_cpu(ppd->dd, &ppd->ibport_data.z_ ##cntr, \
1548 ppd->ibport_data.cntr, vl, \
1549 mode, data); \
1550}
1551
1552def_access_sw_cpu(rc_acks);
1553def_access_sw_cpu(rc_qacks);
1554def_access_sw_cpu(rc_delayed_comp);
1555
1556#define def_access_ibp_counter(cntr) \
1557static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
1558 void *context, int vl, int mode, u64 data) \
1559{ \
1560 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
1561 \
1562 if (vl != CNTR_INVALID_VL) \
1563 return 0; \
1564 \
1565 return read_write_sw(ppd->dd, &ppd->ibport_data.n_ ##cntr, \
1566 mode, data); \
1567}
1568
1569def_access_ibp_counter(loop_pkts);
1570def_access_ibp_counter(rc_resends);
1571def_access_ibp_counter(rnr_naks);
1572def_access_ibp_counter(other_naks);
1573def_access_ibp_counter(rc_timeouts);
1574def_access_ibp_counter(pkt_drops);
1575def_access_ibp_counter(dmawait);
1576def_access_ibp_counter(rc_seqnak);
1577def_access_ibp_counter(rc_dupreq);
1578def_access_ibp_counter(rdma_seq);
1579def_access_ibp_counter(unaligned);
1580def_access_ibp_counter(seq_naks);
1581
1582static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
1583[C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
1584[C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
1585 CNTR_NORMAL),
1586[C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
1587 CNTR_NORMAL),
1588[C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
1589 RCV_TID_FLOW_GEN_MISMATCH_CNT,
1590 CNTR_NORMAL),
77241056
MM		 1591[C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
1592 CNTR_NORMAL),
1593[C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
1594 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
1595[C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
1596 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
1597[C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
1598 CNTR_NORMAL),
1599[C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
1600 CNTR_NORMAL),
1601[C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
1602 CNTR_NORMAL),
1603[C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
1604 CNTR_NORMAL),
1605[C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
1606 CNTR_NORMAL),
1607[C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
1608 CNTR_NORMAL),
1609[C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
1610 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
1611[C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
1612 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
1613[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
1614 CNTR_SYNTH),
1615[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
1616[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
1617 CNTR_SYNTH),
1618[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
1619 CNTR_SYNTH),
1620[C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
1621 CNTR_SYNTH),
1622[C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
1623 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
1624[C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
1625 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
1626 CNTR_SYNTH),
1627[C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
1628 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
1629[C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
1630 CNTR_SYNTH),
1631[C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
1632 CNTR_SYNTH),
1633[C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
1634 CNTR_SYNTH),
1635[C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
1636 CNTR_SYNTH),
1637[C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
1638 CNTR_SYNTH),
1639[C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
1640 CNTR_SYNTH),
1641[C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
1642 CNTR_SYNTH),
1643[C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
1644 CNTR_SYNTH | CNTR_VL),
1645[C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
1646 CNTR_SYNTH | CNTR_VL),
1647[C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
1648[C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
1649 CNTR_SYNTH | CNTR_VL),
1650[C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
1651[C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
1652 CNTR_SYNTH | CNTR_VL),
1653[C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
1654 CNTR_SYNTH),
1655[C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
1656 CNTR_SYNTH | CNTR_VL),
1657[C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
1658 CNTR_SYNTH),
1659[C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
1660 CNTR_SYNTH | CNTR_VL),
1661[C_DC_TOTAL_CRC] =
1662 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
1663 CNTR_SYNTH),
1664[C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
1665 CNTR_SYNTH),
1666[C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
1667 CNTR_SYNTH),
1668[C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
1669 CNTR_SYNTH),
1670[C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
1671 CNTR_SYNTH),
1672[C_DC_CRC_MULT_LN] =
1673 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
1674 CNTR_SYNTH),
1675[C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
1676 CNTR_SYNTH),
1677[C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
1678 CNTR_SYNTH),
1679[C_DC_SEQ_CRC_CNT] =
1680 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
1681 CNTR_SYNTH),
1682[C_DC_ESC0_ONLY_CNT] =
1683 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
1684 CNTR_SYNTH),
1685[C_DC_ESC0_PLUS1_CNT] =
1686 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
1687 CNTR_SYNTH),
1688[C_DC_ESC0_PLUS2_CNT] =
1689 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
1690 CNTR_SYNTH),
1691[C_DC_REINIT_FROM_PEER_CNT] =
1692 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
1693 CNTR_SYNTH),
1694[C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
1695 CNTR_SYNTH),
1696[C_DC_MISC_FLG_CNT] =
1697 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
1698 CNTR_SYNTH),
1699[C_DC_PRF_GOOD_LTP_CNT] =
1700 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
1701[C_DC_PRF_ACCEPTED_LTP_CNT] =
1702 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
1703 CNTR_SYNTH),
1704[C_DC_PRF_RX_FLIT_CNT] =
1705 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
1706[C_DC_PRF_TX_FLIT_CNT] =
1707 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
1708[C_DC_PRF_CLK_CNTR] =
1709 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
1710[C_DC_PG_DBG_FLIT_CRDTS_CNT] =
1711 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
1712[C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
1713 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
1714 CNTR_SYNTH),
1715[C_DC_PG_STS_TX_SBE_CNT] =
1716 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
1717[C_DC_PG_STS_TX_MBE_CNT] =
1718 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
1719 CNTR_SYNTH),
1720[C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
1721 access_sw_cpu_intr),
1722[C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
1723 access_sw_cpu_rcv_limit),
1724[C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
1725 access_sw_vtx_wait),
1726[C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
1727 access_sw_pio_wait),
1728[C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
1729 access_sw_kmem_wait),
b421922e
DL		 1730[C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
1731 access_sw_send_schedule),
77241056
MM		 1732};
1733
1734static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
1735[C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
1736 CNTR_NORMAL),
1737[C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
1738 CNTR_NORMAL),
1739[C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
1740 CNTR_NORMAL),
1741[C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
1742 CNTR_NORMAL),
1743[C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
1744 CNTR_NORMAL),
1745[C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
1746 CNTR_NORMAL),
1747[C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
1748 CNTR_NORMAL),
1749[C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
1750[C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
1751[C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
1752[C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
1753 CNTR_SYNTH | CNTR_VL),
1754[C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
1755 CNTR_SYNTH | CNTR_VL),
1756[C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
1757 CNTR_SYNTH | CNTR_VL),
1758[C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
1759[C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
1760[C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1761 access_sw_link_dn_cnt),
1762[C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1763 access_sw_link_up_cnt),
1764[C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
1765 access_sw_xmit_discards),
1766[C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
1767 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
1768 access_sw_xmit_discards),
1769[C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
1770 access_xmit_constraint_errs),
1771[C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
1772 access_rcv_constraint_errs),
1773[C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
1774[C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
1775[C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
1776[C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
1777[C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
1778[C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
1779[C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
1780[C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
1781[C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
1782[C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
1783[C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
1784[C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
1785[C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
1786 access_sw_cpu_rc_acks),
1787[C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
1788 access_sw_cpu_rc_qacks),
1789[C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
1790 access_sw_cpu_rc_delayed_comp),
1791[OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
1792[OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
1793[OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
1794[OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
1795[OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
1796[OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
1797[OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
1798[OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
1799[OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
1800[OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
1801[OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
1802[OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
1803[OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
1804[OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
1805[OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
1806[OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
1807[OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
1808[OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
1809[OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
1810[OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
1811[OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
1812[OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
1813[OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
1814[OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
1815[OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
1816[OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
1817[OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
1818[OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
1819[OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
1820[OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
1821[OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
1822[OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
1823[OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
1824[OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
1825[OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
1826[OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
1827[OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
1828[OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
1829[OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
1830[OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
1831[OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
1832[OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
1833[OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
1834[OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
1835[OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
1836[OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
1837[OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
1838[OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
1839[OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
1840[OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
1841[OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
1842[OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
1843[OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
1844[OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
1845[OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
1846[OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
1847[OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
1848[OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
1849[OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
1850[OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
1851[OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
1852[OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
1853[OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
1854[OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
1855[OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
1856[OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
1857[OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
1858[OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
1859[OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
1860[OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
1861[OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
1862[OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
1863[OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
1864[OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
1865[OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
1866[OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
1867[OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
1868[OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
1869[OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
1870[OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
1871};
1872
1873/* ======================================================================== */
1874
77241056
MM		 1875/* return true if this is chip revision revision a */
1876int is_ax(struct hfi1_devdata *dd)
1877{
1878 u8 chip_rev_minor =
1879 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
1880 & CCE_REVISION_CHIP_REV_MINOR_MASK;
1881 return (chip_rev_minor & 0xf0) == 0;
1882}
1883
1884/* return true if this is chip revision revision b */
1885int is_bx(struct hfi1_devdata *dd)
1886{
1887 u8 chip_rev_minor =
1888 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
1889 & CCE_REVISION_CHIP_REV_MINOR_MASK;
995deafa 1890 return (chip_rev_minor & 0xF0) == 0x10;
77241056
MM		 1891}
1892
1893/*
1894 * Append string s to buffer buf. Arguments curp and len are the current
1895 * position and remaining length, respectively.
1896 *
1897 * return 0 on success, 1 on out of room
1898 */
1899static int append_str(char *buf, char **curp, int *lenp, const char *s)
1900{
1901 char *p = *curp;
1902 int len = *lenp;
1903 int result = 0; /* success */
1904 char c;
1905
1906 /* add a comma, if first in the buffer */
1907 if (p != buf) {
1908 if (len == 0) {
1909 result = 1; /* out of room */
1910 goto done;
1911 }
1912 *p++ = ',';
1913 len--;
1914 }
1915
1916 /* copy the string */
1917 while ((c = *s++) != 0) {
1918 if (len == 0) {
1919 result = 1; /* out of room */
1920 goto done;
1921 }
1922 *p++ = c;
1923 len--;
1924 }
1925
1926done:
1927 /* write return values */
1928 *curp = p;
1929 *lenp = len;
1930
1931 return result;
1932}
1933
1934/*
1935 * Using the given flag table, print a comma separated string into
1936 * the buffer. End in '*' if the buffer is too short.
1937 */
1938static char *flag_string(char *buf, int buf_len, u64 flags,
1939 struct flag_table *table, int table_size)
1940{
1941 char extra[32];
1942 char *p = buf;
1943 int len = buf_len;
1944 int no_room = 0;
1945 int i;
1946
1947 /* make sure there is at least 2 so we can form "*" */
1948 if (len < 2)
1949 return "";
1950
1951 len--; /* leave room for a nul */
1952 for (i = 0; i < table_size; i++) {
1953 if (flags & table[i].flag) {
1954 no_room = append_str(buf, &p, &len, table[i].str);
1955 if (no_room)
1956 break;
1957 flags &= ~table[i].flag;
1958 }
1959 }
1960
1961 /* any undocumented bits left? */
1962 if (!no_room && flags) {
1963 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
1964 no_room = append_str(buf, &p, &len, extra);
1965 }
1966
1967 /* add * if ran out of room */
1968 if (no_room) {
1969 /* may need to back up to add space for a '*' */
1970 if (len == 0)
1971 --p;
1972 *p++ = '*';
1973 }
1974
1975 /* add final nul - space already allocated above */
1976 *p = 0;
1977 return buf;
1978}
1979
1980/* first 8 CCE error interrupt source names */
1981static const char * const cce_misc_names[] = {
1982 "CceErrInt", /* 0 */
1983 "RxeErrInt", /* 1 */
1984 "MiscErrInt", /* 2 */
1985 "Reserved3", /* 3 */
1986 "PioErrInt", /* 4 */
1987 "SDmaErrInt", /* 5 */
1988 "EgressErrInt", /* 6 */
1989 "TxeErrInt" /* 7 */
1990};
1991
1992/*
1993 * Return the miscellaneous error interrupt name.
1994 */
1995static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
1996{
1997 if (source < ARRAY_SIZE(cce_misc_names))
1998 strncpy(buf, cce_misc_names[source], bsize);
1999 else
2000 snprintf(buf,
2001 bsize,
2002 "Reserved%u",
2003 source + IS_GENERAL_ERR_START);
2004
2005 return buf;
2006}
2007
2008/*
2009 * Return the SDMA engine error interrupt name.
2010 */
2011static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
2012{
2013 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
2014 return buf;
2015}
2016
2017/*
2018 * Return the send context error interrupt name.
2019 */
2020static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
2021{
2022 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
2023 return buf;
2024}
2025
2026static const char * const various_names[] = {
2027 "PbcInt",
2028 "GpioAssertInt",
2029 "Qsfp1Int",
2030 "Qsfp2Int",
2031 "TCritInt"
2032};
2033
2034/*
2035 * Return the various interrupt name.
2036 */
2037static char *is_various_name(char *buf, size_t bsize, unsigned int source)
2038{
2039 if (source < ARRAY_SIZE(various_names))
2040 strncpy(buf, various_names[source], bsize);
2041 else
2042 snprintf(buf, bsize, "Reserved%u", source+IS_VARIOUS_START);
2043 return buf;
2044}
2045
2046/*
2047 * Return the DC interrupt name.
2048 */
2049static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
2050{
2051 static const char * const dc_int_names[] = {
2052 "common",
2053 "lcb",
2054 "8051",
2055 "lbm" /* local block merge */
2056 };
2057
2058 if (source < ARRAY_SIZE(dc_int_names))
2059 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
2060 else
2061 snprintf(buf, bsize, "DCInt%u", source);
2062 return buf;
2063}
2064
2065static const char * const sdma_int_names[] = {
2066 "SDmaInt",
2067 "SdmaIdleInt",
2068 "SdmaProgressInt",
2069};
2070
2071/*
2072 * Return the SDMA engine interrupt name.
2073 */
2074static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
2075{
2076 /* what interrupt */
2077 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
2078 /* which engine */
2079 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
2080
2081 if (likely(what < 3))
2082 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
2083 else
2084 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
2085 return buf;
2086}
2087
2088/*
2089 * Return the receive available interrupt name.
2090 */
2091static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
2092{
2093 snprintf(buf, bsize, "RcvAvailInt%u", source);
2094 return buf;
2095}
2096
2097/*
2098 * Return the receive urgent interrupt name.
2099 */
2100static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
2101{
2102 snprintf(buf, bsize, "RcvUrgentInt%u", source);
2103 return buf;
2104}
2105
2106/*
2107 * Return the send credit interrupt name.
2108 */
2109static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
2110{
2111 snprintf(buf, bsize, "SendCreditInt%u", source);
2112 return buf;
2113}
2114
2115/*
2116 * Return the reserved interrupt name.
2117 */
2118static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
2119{
2120 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
2121 return buf;
2122}
2123
2124static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
2125{
2126 return flag_string(buf, buf_len, flags,
2127 cce_err_status_flags, ARRAY_SIZE(cce_err_status_flags));
2128}
2129
2130static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
2131{
2132 return flag_string(buf, buf_len, flags,
2133 rxe_err_status_flags, ARRAY_SIZE(rxe_err_status_flags));
2134}
2135
2136static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
2137{
2138 return flag_string(buf, buf_len, flags, misc_err_status_flags,
2139 ARRAY_SIZE(misc_err_status_flags));
2140}
2141
2142static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
2143{
2144 return flag_string(buf, buf_len, flags,
2145 pio_err_status_flags, ARRAY_SIZE(pio_err_status_flags));
2146}
2147
2148static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
2149{
2150 return flag_string(buf, buf_len, flags,
2151 sdma_err_status_flags,
2152 ARRAY_SIZE(sdma_err_status_flags));
2153}
2154
2155static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
2156{
2157 return flag_string(buf, buf_len, flags,
2158 egress_err_status_flags, ARRAY_SIZE(egress_err_status_flags));
2159}
2160
2161static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
2162{
2163 return flag_string(buf, buf_len, flags,
2164 egress_err_info_flags, ARRAY_SIZE(egress_err_info_flags));
2165}
2166
2167static char *send_err_status_string(char *buf, int buf_len, u64 flags)
2168{
2169 return flag_string(buf, buf_len, flags,
2170 send_err_status_flags,
2171 ARRAY_SIZE(send_err_status_flags));
2172}
2173
2174static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2175{
2176 char buf[96];
2177
2178 /*
2179 * For most these errors, there is nothing that can be done except
2180 * report or record it.
2181 */
2182 dd_dev_info(dd, "CCE Error: %s\n",
2183 cce_err_status_string(buf, sizeof(buf), reg));
2184
995deafa
MM		 2185 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
2186 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 2187 /* this error requires a manual drop into SPC freeze mode */
2188 /* then a fix up */
2189 start_freeze_handling(dd->pport, FREEZE_SELF);
2190 }
2191}
2192
2193/*
2194 * Check counters for receive errors that do not have an interrupt
2195 * associated with them.
2196 */
2197#define RCVERR_CHECK_TIME 10
2198static void update_rcverr_timer(unsigned long opaque)
2199{
2200 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
2201 struct hfi1_pportdata *ppd = dd->pport;
2202 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
2203
2204 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
2205 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
2206 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
2207 set_link_down_reason(ppd,
2208 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
2209 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
2210 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
2211 }
2212 dd->rcv_ovfl_cnt = (u32) cur_ovfl_cnt;
2213
2214 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
2215}
2216
2217static int init_rcverr(struct hfi1_devdata *dd)
2218{
24523a94 2219 setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
77241056
MM		 2220 /* Assume the hardware counter has been reset */
2221 dd->rcv_ovfl_cnt = 0;
2222 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
2223}
2224
2225static void free_rcverr(struct hfi1_devdata *dd)
2226{
2227 if (dd->rcverr_timer.data)
2228 del_timer_sync(&dd->rcverr_timer);
2229 dd->rcverr_timer.data = 0;
2230}
2231
2232static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2233{
2234 char buf[96];
2235
2236 dd_dev_info(dd, "Receive Error: %s\n",
2237 rxe_err_status_string(buf, sizeof(buf), reg));
2238
2239 if (reg & ALL_RXE_FREEZE_ERR) {
2240 int flags = 0;
2241
2242 /*
2243 * Freeze mode recovery is disabled for the errors
2244 * in RXE_FREEZE_ABORT_MASK
2245 */
995deafa 2246 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
77241056
MM		 2247 flags = FREEZE_ABORT;
2248
2249 start_freeze_handling(dd->pport, flags);
2250 }
2251}
2252
2253static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2254{
2255 char buf[96];
2256
2257 dd_dev_info(dd, "Misc Error: %s",
2258 misc_err_status_string(buf, sizeof(buf), reg));
2259}
2260
2261static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2262{
2263 char buf[96];
2264
2265 dd_dev_info(dd, "PIO Error: %s\n",
2266 pio_err_status_string(buf, sizeof(buf), reg));
2267
2268 if (reg & ALL_PIO_FREEZE_ERR)
2269 start_freeze_handling(dd->pport, 0);
2270}
2271
2272static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2273{
2274 char buf[96];
2275
2276 dd_dev_info(dd, "SDMA Error: %s\n",
2277 sdma_err_status_string(buf, sizeof(buf), reg));
2278
2279 if (reg & ALL_SDMA_FREEZE_ERR)
2280 start_freeze_handling(dd->pport, 0);
2281}
2282
2283static void count_port_inactive(struct hfi1_devdata *dd)
2284{
2285 struct hfi1_pportdata *ppd = dd->pport;
2286
2287 if (ppd->port_xmit_discards < ~(u64)0)
2288 ppd->port_xmit_discards++;
2289}
2290
2291/*
2292 * We have had a "disallowed packet" error during egress. Determine the
2293 * integrity check which failed, and update relevant error counter, etc.
2294 *
2295 * Note that the SEND_EGRESS_ERR_INFO register has only a single
2296 * bit of state per integrity check, and so we can miss the reason for an
2297 * egress error if more than one packet fails the same integrity check
2298 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
2299 */
2300static void handle_send_egress_err_info(struct hfi1_devdata *dd)
2301{
2302 struct hfi1_pportdata *ppd = dd->pport;
2303 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
2304 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
2305 char buf[96];
2306
2307 /* clear down all observed info as quickly as possible after read */
2308 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
2309
2310 dd_dev_info(dd,
2311 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
2312 info, egress_err_info_string(buf, sizeof(buf), info), src);
2313
2314 /* Eventually add other counters for each bit */
2315
2316 if (info & SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK) {
2317 if (ppd->port_xmit_discards < ~(u64)0)
2318 ppd->port_xmit_discards++;
2319 }
2320}
2321
2322/*
2323 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
2324 * register. Does it represent a 'port inactive' error?
2325 */
2326static inline int port_inactive_err(u64 posn)
2327{
2328 return (posn >= SEES(TX_LINKDOWN) &&
2329 posn <= SEES(TX_INCORRECT_LINK_STATE));
2330}
2331
2332/*
2333 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
2334 * register. Does it represent a 'disallowed packet' error?
2335 */
2336static inline int disallowed_pkt_err(u64 posn)
2337{
2338 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
2339 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
2340}
2341
2342static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2343{
2344 u64 reg_copy = reg, handled = 0;
2345 char buf[96];
2346
2347 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
2348 start_freeze_handling(dd->pport, 0);
995deafa 2349 if (is_ax(dd) && (reg &
77241056
MM		 2350 SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK)
2351 && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
2352 start_freeze_handling(dd->pport, 0);
2353
2354 while (reg_copy) {
2355 int posn = fls64(reg_copy);
2356 /*
2357 * fls64() returns a 1-based offset, but we generally
2358 * want 0-based offsets.
2359 */
2360 int shift = posn - 1;
2361
2362 if (port_inactive_err(shift)) {
2363 count_port_inactive(dd);
2364 handled |= (1ULL << shift);
2365 } else if (disallowed_pkt_err(shift)) {
2366 handle_send_egress_err_info(dd);
2367 handled |= (1ULL << shift);
2368 }
2369 clear_bit(shift, (unsigned long *)&reg_copy);
2370 }
2371
2372 reg &= ~handled;
2373
2374 if (reg)
2375 dd_dev_info(dd, "Egress Error: %s\n",
2376 egress_err_status_string(buf, sizeof(buf), reg));
2377}
2378
2379static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
2380{
2381 char buf[96];
2382
2383 dd_dev_info(dd, "Send Error: %s\n",
2384 send_err_status_string(buf, sizeof(buf), reg));
2385
2386}
2387
2388/*
2389 * The maximum number of times the error clear down will loop before
2390 * blocking a repeating error. This value is arbitrary.
2391 */
2392#define MAX_CLEAR_COUNT 20
2393
2394/*
2395 * Clear and handle an error register. All error interrupts are funneled
2396 * through here to have a central location to correctly handle single-
2397 * or multi-shot errors.
2398 *
2399 * For non per-context registers, call this routine with a context value
2400 * of 0 so the per-context offset is zero.
2401 *
2402 * If the handler loops too many times, assume that something is wrong
2403 * and can't be fixed, so mask the error bits.
2404 */
2405static void interrupt_clear_down(struct hfi1_devdata *dd,
2406 u32 context,
2407 const struct err_reg_info *eri)
2408{
2409 u64 reg;
2410 u32 count;
2411
2412 /* read in a loop until no more errors are seen */
2413 count = 0;
2414 while (1) {
2415 reg = read_kctxt_csr(dd, context, eri->status);
2416 if (reg == 0)
2417 break;
2418 write_kctxt_csr(dd, context, eri->clear, reg);
2419 if (likely(eri->handler))
2420 eri->handler(dd, context, reg);
2421 count++;
2422 if (count > MAX_CLEAR_COUNT) {
2423 u64 mask;
2424
2425 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
2426 eri->desc, reg);
2427 /*
2428 * Read-modify-write so any other masked bits
2429 * remain masked.
2430 */
2431 mask = read_kctxt_csr(dd, context, eri->mask);
2432 mask &= ~reg;
2433 write_kctxt_csr(dd, context, eri->mask, mask);
2434 break;
2435 }
2436 }
2437}
2438
2439/*
2440 * CCE block "misc" interrupt. Source is < 16.
2441 */
2442static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
2443{
2444 const struct err_reg_info *eri = &misc_errs[source];
2445
2446 if (eri->handler) {
2447 interrupt_clear_down(dd, 0, eri);
2448 } else {
2449 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
2450 source);
2451 }
2452}
2453
2454static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
2455{
2456 return flag_string(buf, buf_len, flags,
2457 sc_err_status_flags, ARRAY_SIZE(sc_err_status_flags));
2458}
2459
2460/*
2461 * Send context error interrupt. Source (hw_context) is < 160.
2462 *
2463 * All send context errors cause the send context to halt. The normal
2464 * clear-down mechanism cannot be used because we cannot clear the
2465 * error bits until several other long-running items are done first.
2466 * This is OK because with the context halted, nothing else is going
2467 * to happen on it anyway.
2468 */
2469static void is_sendctxt_err_int(struct hfi1_devdata *dd,
2470 unsigned int hw_context)
2471{
2472 struct send_context_info *sci;
2473 struct send_context *sc;
2474 char flags[96];
2475 u64 status;
2476 u32 sw_index;
2477
2478 sw_index = dd->hw_to_sw[hw_context];
2479 if (sw_index >= dd->num_send_contexts) {
2480 dd_dev_err(dd,
2481 "out of range sw index %u for send context %u\n",
2482 sw_index, hw_context);
2483 return;
2484 }
2485 sci = &dd->send_contexts[sw_index];
2486 sc = sci->sc;
2487 if (!sc) {
2488 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
2489 sw_index, hw_context);
2490 return;
2491 }
2492
2493 /* tell the software that a halt has begun */
2494 sc_stop(sc, SCF_HALTED);
2495
2496 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
2497
2498 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
2499 send_context_err_status_string(flags, sizeof(flags), status));
2500
2501 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
2502 handle_send_egress_err_info(dd);
2503
2504 /*
2505 * Automatically restart halted kernel contexts out of interrupt
2506 * context. User contexts must ask the driver to restart the context.
2507 */
2508 if (sc->type != SC_USER)
2509 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
2510}
2511
2512static void handle_sdma_eng_err(struct hfi1_devdata *dd,
2513 unsigned int source, u64 status)
2514{
2515 struct sdma_engine *sde;
2516
2517 sde = &dd->per_sdma[source];
2518#ifdef CONFIG_SDMA_VERBOSITY
2519 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
2520 slashstrip(__FILE__), __LINE__, __func__);
2521 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
2522 sde->this_idx, source, (unsigned long long)status);
2523#endif
2524 sdma_engine_error(sde, status);
2525}
2526
2527/*
2528 * CCE block SDMA error interrupt. Source is < 16.
2529 */
2530static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
2531{
2532#ifdef CONFIG_SDMA_VERBOSITY
2533 struct sdma_engine *sde = &dd->per_sdma[source];
2534
2535 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
2536 slashstrip(__FILE__), __LINE__, __func__);
2537 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
2538 source);
2539 sdma_dumpstate(sde);
2540#endif
2541 interrupt_clear_down(dd, source, &sdma_eng_err);
2542}
2543
2544/*
2545 * CCE block "various" interrupt. Source is < 8.
2546 */
2547static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
2548{
2549 const struct err_reg_info *eri = &various_err[source];
2550
2551 /*
2552 * TCritInt cannot go through interrupt_clear_down()
2553 * because it is not a second tier interrupt. The handler
2554 * should be called directly.
2555 */
2556 if (source == TCRIT_INT_SOURCE)
2557 handle_temp_err(dd);
2558 else if (eri->handler)
2559 interrupt_clear_down(dd, 0, eri);
2560 else
2561 dd_dev_info(dd,
2562 "%s: Unimplemented/reserved interrupt %d\n",
2563 __func__, source);
2564}
2565
2566static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
2567{
2568 /* source is always zero */
2569 struct hfi1_pportdata *ppd = dd->pport;
2570 unsigned long flags;
2571 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
2572
2573 if (reg & QSFP_HFI0_MODPRST_N) {
2574
2575 dd_dev_info(dd, "%s: ModPresent triggered QSFP interrupt\n",
2576 __func__);
2577
2578 if (!qsfp_mod_present(ppd)) {
2579 ppd->driver_link_ready = 0;
2580 /*
2581 * Cable removed, reset all our information about the
2582 * cache and cable capabilities
2583 */
2584
2585 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2586 /*
2587 * We don't set cache_refresh_required here as we expect
2588 * an interrupt when a cable is inserted
2589 */
2590 ppd->qsfp_info.cache_valid = 0;
2591 ppd->qsfp_info.qsfp_interrupt_functional = 0;
2592 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
2593 flags);
2594 write_csr(dd,
2595 dd->hfi1_id ?
2596 ASIC_QSFP2_INVERT :
2597 ASIC_QSFP1_INVERT,
2598 qsfp_int_mgmt);
2599 if (ppd->host_link_state == HLS_DN_POLL) {
2600 /*
2601 * The link is still in POLL. This means
2602 * that the normal link down processing
2603 * will not happen. We have to do it here
2604 * before turning the DC off.
2605 */
2606 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
2607 }
2608 } else {
2609 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2610 ppd->qsfp_info.cache_valid = 0;
2611 ppd->qsfp_info.cache_refresh_required = 1;
2612 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
2613 flags);
2614
2615 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
2616 write_csr(dd,
2617 dd->hfi1_id ?
2618 ASIC_QSFP2_INVERT :
2619 ASIC_QSFP1_INVERT,
2620 qsfp_int_mgmt);
2621 }
2622 }
2623
2624 if (reg & QSFP_HFI0_INT_N) {
2625
2626 dd_dev_info(dd, "%s: IntN triggered QSFP interrupt\n",
2627 __func__);
2628 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
2629 ppd->qsfp_info.check_interrupt_flags = 1;
2630 ppd->qsfp_info.qsfp_interrupt_functional = 1;
2631 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
2632 }
2633
2634 /* Schedule the QSFP work only if there is a cable attached. */
2635 if (qsfp_mod_present(ppd))
2636 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
2637}
2638
2639static int request_host_lcb_access(struct hfi1_devdata *dd)
2640{
2641 int ret;
2642
2643 ret = do_8051_command(dd, HCMD_MISC,
2644 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << LOAD_DATA_FIELD_ID_SHIFT,
2645 NULL);
2646 if (ret != HCMD_SUCCESS) {
2647 dd_dev_err(dd, "%s: command failed with error %d\n",
2648 __func__, ret);
2649 }
2650 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
2651}
2652
2653static int request_8051_lcb_access(struct hfi1_devdata *dd)
2654{
2655 int ret;
2656
2657 ret = do_8051_command(dd, HCMD_MISC,
2658 (u64)HCMD_MISC_GRANT_LCB_ACCESS << LOAD_DATA_FIELD_ID_SHIFT,
2659 NULL);
2660 if (ret != HCMD_SUCCESS) {
2661 dd_dev_err(dd, "%s: command failed with error %d\n",
2662 __func__, ret);
2663 }
2664 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
2665}
2666
2667/*
2668 * Set the LCB selector - allow host access. The DCC selector always
2669 * points to the host.
2670 */
2671static inline void set_host_lcb_access(struct hfi1_devdata *dd)
2672{
2673 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
2674 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK
2675 | DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
2676}
2677
2678/*
2679 * Clear the LCB selector - allow 8051 access. The DCC selector always
2680 * points to the host.
2681 */
2682static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
2683{
2684 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
2685 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
2686}
2687
2688/*
2689 * Acquire LCB access from the 8051. If the host already has access,
2690 * just increment a counter. Otherwise, inform the 8051 that the
2691 * host is taking access.
2692 *
2693 * Returns:
2694 * 0 on success
2695 * -EBUSY if the 8051 has control and cannot be disturbed
2696 * -errno if unable to acquire access from the 8051
2697 */
2698int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
2699{
2700 struct hfi1_pportdata *ppd = dd->pport;
2701 int ret = 0;
2702
2703 /*
2704 * Use the host link state lock so the operation of this routine
2705 * { link state check, selector change, count increment } can occur
2706 * as a unit against a link state change. Otherwise there is a
2707 * race between the state change and the count increment.
2708 */
2709 if (sleep_ok) {
2710 mutex_lock(&ppd->hls_lock);
2711 } else {
951842b0 2712 while (!mutex_trylock(&ppd->hls_lock))
77241056
MM		 2713 udelay(1);
2714 }
2715
2716 /* this access is valid only when the link is up */
2717 if ((ppd->host_link_state & HLS_UP) == 0) {
2718 dd_dev_info(dd, "%s: link state %s not up\n",
2719 __func__, link_state_name(ppd->host_link_state));
2720 ret = -EBUSY;
2721 goto done;
2722 }
2723
2724 if (dd->lcb_access_count == 0) {
2725 ret = request_host_lcb_access(dd);
2726 if (ret) {
2727 dd_dev_err(dd,
2728 "%s: unable to acquire LCB access, err %d\n",
2729 __func__, ret);
2730 goto done;
2731 }
2732 set_host_lcb_access(dd);
2733 }
2734 dd->lcb_access_count++;
2735done:
2736 mutex_unlock(&ppd->hls_lock);
2737 return ret;
2738}
2739
2740/*
2741 * Release LCB access by decrementing the use count. If the count is moving
2742 * from 1 to 0, inform 8051 that it has control back.
2743 *
2744 * Returns:
2745 * 0 on success
2746 * -errno if unable to release access to the 8051
2747 */
2748int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
2749{
2750 int ret = 0;
2751
2752 /*
2753 * Use the host link state lock because the acquire needed it.
2754 * Here, we only need to keep { selector change, count decrement }
2755 * as a unit.
2756 */
2757 if (sleep_ok) {
2758 mutex_lock(&dd->pport->hls_lock);
2759 } else {
951842b0 2760 while (!mutex_trylock(&dd->pport->hls_lock))
77241056
MM		 2761 udelay(1);
2762 }
2763
2764 if (dd->lcb_access_count == 0) {
2765 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
2766 __func__);
2767 goto done;
2768 }
2769
2770 if (dd->lcb_access_count == 1) {
2771 set_8051_lcb_access(dd);
2772 ret = request_8051_lcb_access(dd);
2773 if (ret) {
2774 dd_dev_err(dd,
2775 "%s: unable to release LCB access, err %d\n",
2776 __func__, ret);
2777 /* restore host access if the grant didn't work */
2778 set_host_lcb_access(dd);
2779 goto done;
2780 }
2781 }
2782 dd->lcb_access_count--;
2783done:
2784 mutex_unlock(&dd->pport->hls_lock);
2785 return ret;
2786}
2787
2788/*
2789 * Initialize LCB access variables and state. Called during driver load,
2790 * after most of the initialization is finished.
2791 *
2792 * The DC default is LCB access on for the host. The driver defaults to
2793 * leaving access to the 8051. Assign access now - this constrains the call
2794 * to this routine to be after all LCB set-up is done. In particular, after
2795 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
2796 */
2797static void init_lcb_access(struct hfi1_devdata *dd)
2798{
2799 dd->lcb_access_count = 0;
2800}
2801
2802/*
2803 * Write a response back to a 8051 request.
2804 */
2805static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
2806{
2807 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
2808 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK
2809 | (u64)return_code << DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT
2810 | (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
2811}
2812
2813/*
2814 * Handle requests from the 8051.
2815 */
2816static void handle_8051_request(struct hfi1_devdata *dd)
2817{
2818 u64 reg;
2819 u16 data;
2820 u8 type;
2821
2822 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
2823 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
2824 return; /* no request */
2825
2826 /* zero out COMPLETED so the response is seen */
2827 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
2828
2829 /* extract request details */
2830 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
2831 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
2832 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
2833 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
2834
2835 switch (type) {
2836 case HREQ_LOAD_CONFIG:
2837 case HREQ_SAVE_CONFIG:
2838 case HREQ_READ_CONFIG:
2839 case HREQ_SET_TX_EQ_ABS:
2840 case HREQ_SET_TX_EQ_REL:
2841 case HREQ_ENABLE:
2842 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
2843 type);
2844 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
2845 break;
2846
2847 case HREQ_CONFIG_DONE:
2848 hreq_response(dd, HREQ_SUCCESS, 0);
2849 break;
2850
2851 case HREQ_INTERFACE_TEST:
2852 hreq_response(dd, HREQ_SUCCESS, data);
2853 break;
2854
2855 default:
2856 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
2857 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
2858 break;
2859 }
2860}
2861
2862static void write_global_credit(struct hfi1_devdata *dd,
2863 u8 vau, u16 total, u16 shared)
2864{
2865 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
2866 ((u64)total
2867 << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
2868 | ((u64)shared
2869 << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
2870 | ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
2871}
2872
2873/*
2874 * Set up initial VL15 credits of the remote. Assumes the rest of
2875 * the CM credit registers are zero from a previous global or credit reset .
2876 */
2877void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
2878{
2879 /* leave shared count at zero for both global and VL15 */
2880 write_global_credit(dd, vau, vl15buf, 0);
2881
2882 /* We may need some credits for another VL when sending packets
2883 * with the snoop interface. Dividing it down the middle for VL15
2884 * and VL0 should suffice.
2885 */
2886 if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) {
2887 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1)
2888 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
2889 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1)
2890 << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT);
2891 } else {
2892 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
2893 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
2894 }
2895}
2896
2897/*
2898 * Zero all credit details from the previous connection and
2899 * reset the CM manager's internal counters.
2900 */
2901void reset_link_credits(struct hfi1_devdata *dd)
2902{
2903 int i;
2904
2905 /* remove all previous VL credit limits */
2906 for (i = 0; i < TXE_NUM_DATA_VL; i++)
2907 write_csr(dd, SEND_CM_CREDIT_VL + (8*i), 0);
2908 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
2909 write_global_credit(dd, 0, 0, 0);
2910 /* reset the CM block */
2911 pio_send_control(dd, PSC_CM_RESET);
2912}
2913
2914/* convert a vCU to a CU */
2915static u32 vcu_to_cu(u8 vcu)
2916{
2917 return 1 << vcu;
2918}
2919
2920/* convert a CU to a vCU */
2921static u8 cu_to_vcu(u32 cu)
2922{
2923 return ilog2(cu);
2924}
2925
2926/* convert a vAU to an AU */
2927static u32 vau_to_au(u8 vau)
2928{
2929 return 8 * (1 << vau);
2930}
2931
2932static void set_linkup_defaults(struct hfi1_pportdata *ppd)
2933{
2934 ppd->sm_trap_qp = 0x0;
2935 ppd->sa_qp = 0x1;
2936}
2937
2938/*
2939 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
2940 */
2941static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
2942{
2943 u64 reg;
2944
2945 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
2946 write_csr(dd, DC_LCB_CFG_RUN, 0);
2947 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
2948 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
2949 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
2950 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
2951 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
2952 reg = read_csr(dd, DCC_CFG_RESET);
2953 write_csr(dd, DCC_CFG_RESET,
2954 reg
2955 | (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT)
2956 | (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
2957 (void) read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
2958 if (!abort) {
2959 udelay(1); /* must hold for the longer of 16cclks or 20ns */
2960 write_csr(dd, DCC_CFG_RESET, reg);
2961 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
2962 }
2963}
2964
2965/*
2966 * This routine should be called after the link has been transitioned to
2967 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
2968 * reset).
2969 *
2970 * The expectation is that the caller of this routine would have taken
2971 * care of properly transitioning the link into the correct state.
2972 */
2973static void dc_shutdown(struct hfi1_devdata *dd)
2974{
2975 unsigned long flags;
2976
2977 spin_lock_irqsave(&dd->dc8051_lock, flags);
2978 if (dd->dc_shutdown) {
2979 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
2980 return;
2981 }
2982 dd->dc_shutdown = 1;
2983 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
2984 /* Shutdown the LCB */
2985 lcb_shutdown(dd, 1);
2986 /* Going to OFFLINE would have causes the 8051 to put the
2987 * SerDes into reset already. Just need to shut down the 8051,
2988 * itself. */
2989 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
2990}
2991
2992/* Calling this after the DC has been brought out of reset should not
2993 * do any damage. */
2994static void dc_start(struct hfi1_devdata *dd)
2995{
2996 unsigned long flags;
2997 int ret;
2998
2999 spin_lock_irqsave(&dd->dc8051_lock, flags);
3000 if (!dd->dc_shutdown)
3001 goto done;
3002 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
3003 /* Take the 8051 out of reset */
3004 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
3005 /* Wait until 8051 is ready */
3006 ret = wait_fm_ready(dd, TIMEOUT_8051_START);
3007 if (ret) {
3008 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
3009 __func__);
3010 }
3011 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
3012 write_csr(dd, DCC_CFG_RESET, 0x10);
3013 /* lcb_shutdown() with abort=1 does not restore these */
3014 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
3015 spin_lock_irqsave(&dd->dc8051_lock, flags);
3016 dd->dc_shutdown = 0;
3017done:
3018 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
3019}
3020
3021/*
3022 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
3023 */
3024static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
3025{
3026 u64 rx_radr, tx_radr;
3027 u32 version;
3028
3029 if (dd->icode != ICODE_FPGA_EMULATION)
3030 return;
3031
3032 /*
3033 * These LCB defaults on emulator _s are good, nothing to do here:
3034 * LCB_CFG_TX_FIFOS_RADR
3035 * LCB_CFG_RX_FIFOS_RADR
3036 * LCB_CFG_LN_DCLK
3037 * LCB_CFG_IGNORE_LOST_RCLK
3038 */
3039 if (is_emulator_s(dd))
3040 return;
3041 /* else this is _p */
3042
3043 version = emulator_rev(dd);
995deafa 3044 if (!is_ax(dd))
77241056
MM		 3045 version = 0x2d; /* all B0 use 0x2d or higher settings */
3046
3047 if (version <= 0x12) {
3048 /* release 0x12 and below */
3049
3050 /*
3051 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
3052 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
3053 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
3054 */
3055 rx_radr =
3056 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3057 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3058 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3059 /*
3060 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
3061 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
3062 */
3063 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3064 } else if (version <= 0x18) {
3065 /* release 0x13 up to 0x18 */
3066 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
3067 rx_radr =
3068 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3069 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3070 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3071 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3072 } else if (version == 0x19) {
3073 /* release 0x19 */
3074 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
3075 rx_radr =
3076 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3077 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3078 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3079 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3080 } else if (version == 0x1a) {
3081 /* release 0x1a */
3082 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
3083 rx_radr =
3084 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3085 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3086 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3087 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3088 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
3089 } else {
3090 /* release 0x1b and higher */
3091 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
3092 rx_radr =
3093 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
3094 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
3095 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
3096 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
3097 }
3098
3099 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
3100 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
3101 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
3102 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
3103 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
3104}
3105
3106/*
3107 * Handle a SMA idle message
3108 *
3109 * This is a work-queue function outside of the interrupt.
3110 */
3111void handle_sma_message(struct work_struct *work)
3112{
3113 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3114 sma_message_work);
3115 struct hfi1_devdata *dd = ppd->dd;
3116 u64 msg;
3117 int ret;
3118
3119 /* msg is bytes 1-4 of the 40-bit idle message - the command code
3120 is stripped off */
3121 ret = read_idle_sma(dd, &msg);
3122 if (ret)
3123 return;
3124 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
3125 /*
3126 * React to the SMA message. Byte[1] (0 for us) is the command.
3127 */
3128 switch (msg & 0xff) {
3129 case SMA_IDLE_ARM:
3130 /*
3131 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
3132 * State Transitions
3133 *
3134 * Only expected in INIT or ARMED, discard otherwise.
3135 */
3136 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
3137 ppd->neighbor_normal = 1;
3138 break;
3139 case SMA_IDLE_ACTIVE:
3140 /*
3141 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
3142 * State Transitions
3143 *
3144 * Can activate the node. Discard otherwise.
3145 */
3146 if (ppd->host_link_state == HLS_UP_ARMED
3147 && ppd->is_active_optimize_enabled) {
3148 ppd->neighbor_normal = 1;
3149 ret = set_link_state(ppd, HLS_UP_ACTIVE);
3150 if (ret)
3151 dd_dev_err(
3152 dd,
3153 "%s: received Active SMA idle message, couldn't set link to Active\n",
3154 __func__);
3155 }
3156 break;
3157 default:
3158 dd_dev_err(dd,
3159 "%s: received unexpected SMA idle message 0x%llx\n",
3160 __func__, msg);
3161 break;
3162 }
3163}
3164
3165static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
3166{
3167 u64 rcvctrl;
3168 unsigned long flags;
3169
3170 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
3171 rcvctrl = read_csr(dd, RCV_CTRL);
3172 rcvctrl |= add;
3173 rcvctrl &= ~clear;
3174 write_csr(dd, RCV_CTRL, rcvctrl);
3175 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
3176}
3177
3178static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
3179{
3180 adjust_rcvctrl(dd, add, 0);
3181}
3182
3183static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
3184{
3185 adjust_rcvctrl(dd, 0, clear);
3186}
3187
3188/*
3189 * Called from all interrupt handlers to start handling an SPC freeze.
3190 */
3191void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
3192{
3193 struct hfi1_devdata *dd = ppd->dd;
3194 struct send_context *sc;
3195 int i;
3196
3197 if (flags & FREEZE_SELF)
3198 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
3199
3200 /* enter frozen mode */
3201 dd->flags |= HFI1_FROZEN;
3202
3203 /* notify all SDMA engines that they are going into a freeze */
3204 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
3205
3206 /* do halt pre-handling on all enabled send contexts */
3207 for (i = 0; i < dd->num_send_contexts; i++) {
3208 sc = dd->send_contexts[i].sc;
3209 if (sc && (sc->flags & SCF_ENABLED))
3210 sc_stop(sc, SCF_FROZEN | SCF_HALTED);
3211 }
3212
3213 /* Send context are frozen. Notify user space */
3214 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
3215
3216 if (flags & FREEZE_ABORT) {
3217 dd_dev_err(dd,
3218 "Aborted freeze recovery. Please REBOOT system\n");
3219 return;
3220 }
3221 /* queue non-interrupt handler */
3222 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
3223}
3224
3225/*
3226 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
3227 * depending on the "freeze" parameter.
3228 *
3229 * No need to return an error if it times out, our only option
3230 * is to proceed anyway.
3231 */
3232static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
3233{
3234 unsigned long timeout;
3235 u64 reg;
3236
3237 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
3238 while (1) {
3239 reg = read_csr(dd, CCE_STATUS);
3240 if (freeze) {
3241 /* waiting until all indicators are set */
3242 if ((reg & ALL_FROZE) == ALL_FROZE)
3243 return; /* all done */
3244 } else {
3245 /* waiting until all indicators are clear */
3246 if ((reg & ALL_FROZE) == 0)
3247 return; /* all done */
3248 }
3249
3250 if (time_after(jiffies, timeout)) {
3251 dd_dev_err(dd,
3252 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
3253 freeze ? "" : "un",
3254 reg & ALL_FROZE,
3255 freeze ? ALL_FROZE : 0ull);
3256 return;
3257 }
3258 usleep_range(80, 120);
3259 }
3260}
3261
3262/*
3263 * Do all freeze handling for the RXE block.
3264 */
3265static void rxe_freeze(struct hfi1_devdata *dd)
3266{
3267 int i;
3268
3269 /* disable port */
3270 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3271
3272 /* disable all receive contexts */
3273 for (i = 0; i < dd->num_rcv_contexts; i++)
3274 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
3275}
3276
3277/*
3278 * Unfreeze handling for the RXE block - kernel contexts only.
3279 * This will also enable the port. User contexts will do unfreeze
3280 * handling on a per-context basis as they call into the driver.
3281 *
3282 */
3283static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
3284{
3285 int i;
3286
3287 /* enable all kernel contexts */
3288 for (i = 0; i < dd->n_krcv_queues; i++)
3289 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB, i);
3290
3291 /* enable port */
3292 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3293}
3294
3295/*
3296 * Non-interrupt SPC freeze handling.
3297 *
3298 * This is a work-queue function outside of the triggering interrupt.
3299 */
3300void handle_freeze(struct work_struct *work)
3301{
3302 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3303 freeze_work);
3304 struct hfi1_devdata *dd = ppd->dd;
3305
3306 /* wait for freeze indicators on all affected blocks */
3307 dd_dev_info(dd, "Entering SPC freeze\n");
3308 wait_for_freeze_status(dd, 1);
3309
3310 /* SPC is now frozen */
3311
3312 /* do send PIO freeze steps */
3313 pio_freeze(dd);
3314
3315 /* do send DMA freeze steps */
3316 sdma_freeze(dd);
3317
3318 /* do send egress freeze steps - nothing to do */
3319
3320 /* do receive freeze steps */
3321 rxe_freeze(dd);
3322
3323 /*
3324 * Unfreeze the hardware - clear the freeze, wait for each
3325 * block's frozen bit to clear, then clear the frozen flag.
3326 */
3327 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
3328 wait_for_freeze_status(dd, 0);
3329
995deafa 3330 if (is_ax(dd)) {
77241056
MM		 3331 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
3332 wait_for_freeze_status(dd, 1);
3333 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
3334 wait_for_freeze_status(dd, 0);
3335 }
3336
3337 /* do send PIO unfreeze steps for kernel contexts */
3338 pio_kernel_unfreeze(dd);
3339
3340 /* do send DMA unfreeze steps */
3341 sdma_unfreeze(dd);
3342
3343 /* do send egress unfreeze steps - nothing to do */
3344
3345 /* do receive unfreeze steps for kernel contexts */
3346 rxe_kernel_unfreeze(dd);
3347
3348 /*
3349 * The unfreeze procedure touches global device registers when
3350 * it disables and re-enables RXE. Mark the device unfrozen
3351 * after all that is done so other parts of the driver waiting
3352 * for the device to unfreeze don't do things out of order.
3353 *
3354 * The above implies that the meaning of HFI1_FROZEN flag is
3355 * "Device has gone into freeze mode and freeze mode handling
3356 * is still in progress."
3357 *
3358 * The flag will be removed when freeze mode processing has
3359 * completed.
3360 */
3361 dd->flags &= ~HFI1_FROZEN;
3362 wake_up(&dd->event_queue);
3363
3364 /* no longer frozen */
3365 dd_dev_err(dd, "Exiting SPC freeze\n");
3366}
3367
3368/*
3369 * Handle a link up interrupt from the 8051.
3370 *
3371 * This is a work-queue function outside of the interrupt.
3372 */
3373void handle_link_up(struct work_struct *work)
3374{
3375 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3376 link_up_work);
3377 set_link_state(ppd, HLS_UP_INIT);
3378
3379 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
3380 read_ltp_rtt(ppd->dd);
3381 /*
3382 * OPA specifies that certain counters are cleared on a transition
3383 * to link up, so do that.
3384 */
3385 clear_linkup_counters(ppd->dd);
3386 /*
3387 * And (re)set link up default values.
3388 */
3389 set_linkup_defaults(ppd);
3390
3391 /* enforce link speed enabled */
3392 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
3393 /* oops - current speed is not enabled, bounce */
3394 dd_dev_err(ppd->dd,
3395 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
3396 ppd->link_speed_active, ppd->link_speed_enabled);
3397 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
3398 OPA_LINKDOWN_REASON_SPEED_POLICY);
3399 set_link_state(ppd, HLS_DN_OFFLINE);
3400 start_link(ppd);
3401 }
3402}
3403
3404/* Several pieces of LNI information were cached for SMA in ppd.
3405 * Reset these on link down */
3406static void reset_neighbor_info(struct hfi1_pportdata *ppd)
3407{
3408 ppd->neighbor_guid = 0;
3409 ppd->neighbor_port_number = 0;
3410 ppd->neighbor_type = 0;
3411 ppd->neighbor_fm_security = 0;
3412}
3413
3414/*
3415 * Handle a link down interrupt from the 8051.
3416 *
3417 * This is a work-queue function outside of the interrupt.
3418 */
3419void handle_link_down(struct work_struct *work)
3420{
3421 u8 lcl_reason, neigh_reason = 0;
3422 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3423 link_down_work);
3424
3425 /* go offline first, then deal with reasons */
3426 set_link_state(ppd, HLS_DN_OFFLINE);
3427
3428 lcl_reason = 0;
3429 read_planned_down_reason_code(ppd->dd, &neigh_reason);
3430
3431 /*
3432 * If no reason, assume peer-initiated but missed
3433 * LinkGoingDown idle flits.
3434 */
3435 if (neigh_reason == 0)
3436 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
3437
3438 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
3439
3440 reset_neighbor_info(ppd);
3441
3442 /* disable the port */
3443 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
3444
3445 /* If there is no cable attached, turn the DC off. Otherwise,
3446 * start the link bring up. */
3447 if (!qsfp_mod_present(ppd))
3448 dc_shutdown(ppd->dd);
3449 else
3450 start_link(ppd);
3451}
3452
3453void handle_link_bounce(struct work_struct *work)
3454{
3455 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3456 link_bounce_work);
3457
3458 /*
3459 * Only do something if the link is currently up.
3460 */
3461 if (ppd->host_link_state & HLS_UP) {
3462 set_link_state(ppd, HLS_DN_OFFLINE);
3463 start_link(ppd);
3464 } else {
3465 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
3466 __func__, link_state_name(ppd->host_link_state));
3467 }
3468}
3469
3470/*
3471 * Mask conversion: Capability exchange to Port LTP. The capability
3472 * exchange has an implicit 16b CRC that is mandatory.
3473 */
3474static int cap_to_port_ltp(int cap)
3475{
3476 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
3477
3478 if (cap & CAP_CRC_14B)
3479 port_ltp |= PORT_LTP_CRC_MODE_14;
3480 if (cap & CAP_CRC_48B)
3481 port_ltp |= PORT_LTP_CRC_MODE_48;
3482 if (cap & CAP_CRC_12B_16B_PER_LANE)
3483 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
3484
3485 return port_ltp;
3486}
3487
3488/*
3489 * Convert an OPA Port LTP mask to capability mask
3490 */
3491int port_ltp_to_cap(int port_ltp)
3492{
3493 int cap_mask = 0;
3494
3495 if (port_ltp & PORT_LTP_CRC_MODE_14)
3496 cap_mask |= CAP_CRC_14B;
3497 if (port_ltp & PORT_LTP_CRC_MODE_48)
3498 cap_mask |= CAP_CRC_48B;
3499 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
3500 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
3501
3502 return cap_mask;
3503}
3504
3505/*
3506 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
3507 */
3508static int lcb_to_port_ltp(int lcb_crc)
3509{
3510 int port_ltp = 0;
3511
3512 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
3513 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
3514 else if (lcb_crc == LCB_CRC_48B)
3515 port_ltp = PORT_LTP_CRC_MODE_48;
3516 else if (lcb_crc == LCB_CRC_14B)
3517 port_ltp = PORT_LTP_CRC_MODE_14;
3518 else
3519 port_ltp = PORT_LTP_CRC_MODE_16;
3520
3521 return port_ltp;
3522}
3523
3524/*
3525 * Our neighbor has indicated that we are allowed to act as a fabric
3526 * manager, so place the full management partition key in the second
3527 * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
3528 * that we should already have the limited management partition key in
3529 * array element 1, and also that the port is not yet up when
3530 * add_full_mgmt_pkey() is invoked.
3531 */
3532static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
3533{
3534 struct hfi1_devdata *dd = ppd->dd;
3535
3536 /* Sanity check - ppd->pkeys[2] should be 0 */
3537 if (ppd->pkeys[2] != 0)
3538 dd_dev_err(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
3539 __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
3540 ppd->pkeys[2] = FULL_MGMT_P_KEY;
3541 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
3542}
3543
3544/*
3545 * Convert the given link width to the OPA link width bitmask.
3546 */
3547static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
3548{
3549 switch (width) {
3550 case 0:
3551 /*
3552 * Simulator and quick linkup do not set the width.
3553 * Just set it to 4x without complaint.
3554 */
3555 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
3556 return OPA_LINK_WIDTH_4X;
3557 return 0; /* no lanes up */
3558 case 1: return OPA_LINK_WIDTH_1X;
3559 case 2: return OPA_LINK_WIDTH_2X;
3560 case 3: return OPA_LINK_WIDTH_3X;
3561 default:
3562 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
3563 __func__, width);
3564 /* fall through */
3565 case 4: return OPA_LINK_WIDTH_4X;
3566 }
3567}
3568
3569/*
3570 * Do a population count on the bottom nibble.
3571 */
3572static const u8 bit_counts[16] = {
3573 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
3574};
3575static inline u8 nibble_to_count(u8 nibble)
3576{
3577 return bit_counts[nibble & 0xf];
3578}
3579
3580/*
3581 * Read the active lane information from the 8051 registers and return
3582 * their widths.
3583 *
3584 * Active lane information is found in these 8051 registers:
3585 * enable_lane_tx
3586 * enable_lane_rx
3587 */
3588static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
3589 u16 *rx_width)
3590{
3591 u16 tx, rx;
3592 u8 enable_lane_rx;
3593 u8 enable_lane_tx;
3594 u8 tx_polarity_inversion;
3595 u8 rx_polarity_inversion;
3596 u8 max_rate;
3597
3598 /* read the active lanes */
3599 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
3600 &rx_polarity_inversion, &max_rate);
3601 read_local_lni(dd, &enable_lane_rx);
3602
3603 /* convert to counts */
3604 tx = nibble_to_count(enable_lane_tx);
3605 rx = nibble_to_count(enable_lane_rx);
3606
3607 /*
3608 * Set link_speed_active here, overriding what was set in
3609 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
3610 * set the max_rate field in handle_verify_cap until v0.19.
3611 */
3612 if ((dd->icode == ICODE_RTL_SILICON)
3613 && (dd->dc8051_ver < dc8051_ver(0, 19))) {
3614 /* max_rate: 0 = 12.5G, 1 = 25G */
3615 switch (max_rate) {
3616 case 0:
3617 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
3618 break;
3619 default:
3620 dd_dev_err(dd,
3621 "%s: unexpected max rate %d, using 25Gb\n",
3622 __func__, (int)max_rate);
3623 /* fall through */
3624 case 1:
3625 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
3626 break;
3627 }
3628 }
3629
3630 dd_dev_info(dd,
3631 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
3632 enable_lane_tx, tx, enable_lane_rx, rx);
3633 *tx_width = link_width_to_bits(dd, tx);
3634 *rx_width = link_width_to_bits(dd, rx);
3635}
3636
3637/*
3638 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
3639 * Valid after the end of VerifyCap and during LinkUp. Does not change
3640 * after link up. I.e. look elsewhere for downgrade information.
3641 *
3642 * Bits are:
3643 * + bits [7:4] contain the number of active transmitters
3644 * + bits [3:0] contain the number of active receivers
3645 * These are numbers 1 through 4 and can be different values if the
3646 * link is asymmetric.
3647 *
3648 * verify_cap_local_fm_link_width[0] retains its original value.
3649 */
3650static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
3651 u16 *rx_width)
3652{
3653 u16 widths, tx, rx;
3654 u8 misc_bits, local_flags;
3655 u16 active_tx, active_rx;
3656
3657 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
3658 tx = widths >> 12;
3659 rx = (widths >> 8) & 0xf;
3660
3661 *tx_width = link_width_to_bits(dd, tx);
3662 *rx_width = link_width_to_bits(dd, rx);
3663
3664 /* print the active widths */
3665 get_link_widths(dd, &active_tx, &active_rx);
3666}
3667
3668/*
3669 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
3670 * hardware information when the link first comes up.
3671 *
3672 * The link width is not available until after VerifyCap.AllFramesReceived
3673 * (the trigger for handle_verify_cap), so this is outside that routine
3674 * and should be called when the 8051 signals linkup.
3675 */
3676void get_linkup_link_widths(struct hfi1_pportdata *ppd)
3677{
3678 u16 tx_width, rx_width;
3679
3680 /* get end-of-LNI link widths */
3681 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
3682
3683 /* use tx_width as the link is supposed to be symmetric on link up */
3684 ppd->link_width_active = tx_width;
3685 /* link width downgrade active (LWD.A) starts out matching LW.A */
3686 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
3687 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
3688 /* per OPA spec, on link up LWD.E resets to LWD.S */
3689 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
3690 /* cache the active egress rate (units {10^6 bits/sec]) */
3691 ppd->current_egress_rate = active_egress_rate(ppd);
3692}
3693
3694/*
3695 * Handle a verify capabilities interrupt from the 8051.
3696 *
3697 * This is a work-queue function outside of the interrupt.
3698 */
3699void handle_verify_cap(struct work_struct *work)
3700{
3701 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3702 link_vc_work);
3703 struct hfi1_devdata *dd = ppd->dd;
3704 u64 reg;
3705 u8 power_management;
3706 u8 continious;
3707 u8 vcu;
3708 u8 vau;
3709 u8 z;
3710 u16 vl15buf;
3711 u16 link_widths;
3712 u16 crc_mask;
3713 u16 crc_val;
3714 u16 device_id;
3715 u16 active_tx, active_rx;
3716 u8 partner_supported_crc;
3717 u8 remote_tx_rate;
3718 u8 device_rev;
3719
3720 set_link_state(ppd, HLS_VERIFY_CAP);
3721
3722 lcb_shutdown(dd, 0);
3723 adjust_lcb_for_fpga_serdes(dd);
3724
3725 /*
3726 * These are now valid:
3727 * remote VerifyCap fields in the general LNI config
3728 * CSR DC8051_STS_REMOTE_GUID
3729 * CSR DC8051_STS_REMOTE_NODE_TYPE
3730 * CSR DC8051_STS_REMOTE_FM_SECURITY
3731 * CSR DC8051_STS_REMOTE_PORT_NO
3732 */
3733
3734 read_vc_remote_phy(dd, &power_management, &continious);
3735 read_vc_remote_fabric(
3736 dd,
3737 &vau,
3738 &z,
3739 &vcu,
3740 &vl15buf,
3741 &partner_supported_crc);
3742 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
3743 read_remote_device_id(dd, &device_id, &device_rev);
3744 /*
3745 * And the 'MgmtAllowed' information, which is exchanged during
3746 * LNI, is also be available at this point.
3747 */
3748 read_mgmt_allowed(dd, &ppd->mgmt_allowed);
3749 /* print the active widths */
3750 get_link_widths(dd, &active_tx, &active_rx);
3751 dd_dev_info(dd,
3752 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
3753 (int)power_management, (int)continious);
3754 dd_dev_info(dd,
3755 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
3756 (int)vau,
3757 (int)z,
3758 (int)vcu,
3759 (int)vl15buf,
3760 (int)partner_supported_crc);
3761 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
3762 (u32)remote_tx_rate, (u32)link_widths);
3763 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
3764 (u32)device_id, (u32)device_rev);
3765 /*
3766 * The peer vAU value just read is the peer receiver value. HFI does
3767 * not support a transmit vAU of 0 (AU == 8). We advertised that
3768 * with Z=1 in the fabric capabilities sent to the peer. The peer
3769 * will see our Z=1, and, if it advertised a vAU of 0, will move its
3770 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
3771 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
3772 * subject to the Z value exception.
3773 */
3774 if (vau == 0)
3775 vau = 1;
3776 set_up_vl15(dd, vau, vl15buf);
3777
3778 /* set up the LCB CRC mode */
3779 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
3780
3781 /* order is important: use the lowest bit in common */
3782 if (crc_mask & CAP_CRC_14B)
3783 crc_val = LCB_CRC_14B;
3784 else if (crc_mask & CAP_CRC_48B)
3785 crc_val = LCB_CRC_48B;
3786 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
3787 crc_val = LCB_CRC_12B_16B_PER_LANE;
3788 else
3789 crc_val = LCB_CRC_16B;
3790
3791 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
3792 write_csr(dd, DC_LCB_CFG_CRC_MODE,
3793 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
3794
3795 /* set (14b only) or clear sideband credit */
3796 reg = read_csr(dd, SEND_CM_CTRL);
3797 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
3798 write_csr(dd, SEND_CM_CTRL,
3799 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
3800 } else {
3801 write_csr(dd, SEND_CM_CTRL,
3802 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
3803 }
3804
3805 ppd->link_speed_active = 0; /* invalid value */
3806 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
3807 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
3808 switch (remote_tx_rate) {
3809 case 0:
3810 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
3811 break;
3812 case 1:
3813 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3814 break;
3815 }
3816 } else {
3817 /* actual rate is highest bit of the ANDed rates */
3818 u8 rate = remote_tx_rate & ppd->local_tx_rate;
3819
3820 if (rate & 2)
3821 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3822 else if (rate & 1)
3823 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
3824 }
3825 if (ppd->link_speed_active == 0) {
3826 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
3827 __func__, (int)remote_tx_rate);
3828 ppd->link_speed_active = OPA_LINK_SPEED_25G;
3829 }
3830
3831 /*
3832 * Cache the values of the supported, enabled, and active
3833 * LTP CRC modes to return in 'portinfo' queries. But the bit
3834 * flags that are returned in the portinfo query differ from
3835 * what's in the link_crc_mask, crc_sizes, and crc_val
3836 * variables. Convert these here.
3837 */
3838 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
3839 /* supported crc modes */
3840 ppd->port_ltp_crc_mode |=
3841 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
3842 /* enabled crc modes */
3843 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
3844 /* active crc mode */
3845
3846 /* set up the remote credit return table */
3847 assign_remote_cm_au_table(dd, vcu);
3848
3849 /*
3850 * The LCB is reset on entry to handle_verify_cap(), so this must
3851 * be applied on every link up.
3852 *
3853 * Adjust LCB error kill enable to kill the link if
3854 * these RBUF errors are seen:
3855 * REPLAY_BUF_MBE_SMASK
3856 * FLIT_INPUT_BUF_MBE_SMASK
3857 */
995deafa 3858 if (is_ax(dd)) { /* fixed in B0 */
77241056
MM		 3859 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
3860 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
3861 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
3862 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
3863 }
3864
3865 /* pull LCB fifos out of reset - all fifo clocks must be stable */
3866 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
3867
3868 /* give 8051 access to the LCB CSRs */
3869 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
3870 set_8051_lcb_access(dd);
3871
3872 ppd->neighbor_guid =
3873 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
3874 ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
3875 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
3876 ppd->neighbor_type =
3877 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
3878 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
3879 ppd->neighbor_fm_security =
3880 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
3881 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
3882 dd_dev_info(dd,
3883 "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
3884 ppd->neighbor_guid, ppd->neighbor_type,
3885 ppd->mgmt_allowed, ppd->neighbor_fm_security);
3886 if (ppd->mgmt_allowed)
3887 add_full_mgmt_pkey(ppd);
3888
3889 /* tell the 8051 to go to LinkUp */
3890 set_link_state(ppd, HLS_GOING_UP);
3891}
3892
3893/*
3894 * Apply the link width downgrade enabled policy against the current active
3895 * link widths.
3896 *
3897 * Called when the enabled policy changes or the active link widths change.
3898 */
3899void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
3900{
77241056 3901 int do_bounce = 0;
323fd785
DL		 3902 int tries;
3903 u16 lwde;
77241056
MM		 3904 u16 tx, rx;
3905
323fd785
DL		 3906 /* use the hls lock to avoid a race with actual link up */
3907 tries = 0;
3908retry:
77241056
MM		 3909 mutex_lock(&ppd->hls_lock);
3910 /* only apply if the link is up */
323fd785
DL		 3911 if (!(ppd->host_link_state & HLS_UP)) {
3912 /* still going up..wait and retry */
3913 if (ppd->host_link_state & HLS_GOING_UP) {
3914 if (++tries < 1000) {
3915 mutex_unlock(&ppd->hls_lock);
3916 usleep_range(100, 120); /* arbitrary */
3917 goto retry;
3918 }
3919 dd_dev_err(ppd->dd,
3920 "%s: giving up waiting for link state change\n",
3921 __func__);
3922 }
3923 goto done;
3924 }
3925
3926 lwde = ppd->link_width_downgrade_enabled;
77241056
MM		 3927
3928 if (refresh_widths) {
3929 get_link_widths(ppd->dd, &tx, &rx);
3930 ppd->link_width_downgrade_tx_active = tx;
3931 ppd->link_width_downgrade_rx_active = rx;
3932 }
3933
3934 if (lwde == 0) {
3935 /* downgrade is disabled */
3936
3937 /* bounce if not at starting active width */
3938 if ((ppd->link_width_active !=
3939 ppd->link_width_downgrade_tx_active)
3940 || (ppd->link_width_active !=
3941 ppd->link_width_downgrade_rx_active)) {
3942 dd_dev_err(ppd->dd,
3943 "Link downgrade is disabled and link has downgraded, downing link\n");
3944 dd_dev_err(ppd->dd,
3945 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
3946 ppd->link_width_active,
3947 ppd->link_width_downgrade_tx_active,
3948 ppd->link_width_downgrade_rx_active);
3949 do_bounce = 1;
3950 }
3951 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0
3952 || (lwde & ppd->link_width_downgrade_rx_active) == 0) {
3953 /* Tx or Rx is outside the enabled policy */
3954 dd_dev_err(ppd->dd,
3955 "Link is outside of downgrade allowed, downing link\n");
3956 dd_dev_err(ppd->dd,
3957 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
3958 lwde,
3959 ppd->link_width_downgrade_tx_active,
3960 ppd->link_width_downgrade_rx_active);
3961 do_bounce = 1;
3962 }
3963
323fd785
DL		 3964done:
3965 mutex_unlock(&ppd->hls_lock);
3966
77241056
MM		 3967 if (do_bounce) {
3968 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
3969 OPA_LINKDOWN_REASON_WIDTH_POLICY);
3970 set_link_state(ppd, HLS_DN_OFFLINE);
3971 start_link(ppd);
3972 }
3973}
3974
3975/*
3976 * Handle a link downgrade interrupt from the 8051.
3977 *
3978 * This is a work-queue function outside of the interrupt.
3979 */
3980void handle_link_downgrade(struct work_struct *work)
3981{
3982 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
3983 link_downgrade_work);
3984
3985 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
3986 apply_link_downgrade_policy(ppd, 1);
3987}
3988
3989static char *dcc_err_string(char *buf, int buf_len, u64 flags)
3990{
3991 return flag_string(buf, buf_len, flags, dcc_err_flags,
3992 ARRAY_SIZE(dcc_err_flags));
3993}
3994
3995static char *lcb_err_string(char *buf, int buf_len, u64 flags)
3996{
3997 return flag_string(buf, buf_len, flags, lcb_err_flags,
3998 ARRAY_SIZE(lcb_err_flags));
3999}
4000
4001static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
4002{
4003 return flag_string(buf, buf_len, flags, dc8051_err_flags,
4004 ARRAY_SIZE(dc8051_err_flags));
4005}
4006
4007static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
4008{
4009 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
4010 ARRAY_SIZE(dc8051_info_err_flags));
4011}
4012
4013static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
4014{
4015 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
4016 ARRAY_SIZE(dc8051_info_host_msg_flags));
4017}
4018
4019static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
4020{
4021 struct hfi1_pportdata *ppd = dd->pport;
4022 u64 info, err, host_msg;
4023 int queue_link_down = 0;
4024 char buf[96];
4025
4026 /* look at the flags */
4027 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
4028 /* 8051 information set by firmware */
4029 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
4030 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
4031 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
4032 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
4033 host_msg = (info >>
4034 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
4035 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
4036
4037 /*
4038 * Handle error flags.
4039 */
4040 if (err & FAILED_LNI) {
4041 /*
4042 * LNI error indications are cleared by the 8051
4043 * only when starting polling. Only pay attention
4044 * to them when in the states that occur during
4045 * LNI.
4046 */
4047 if (ppd->host_link_state
4048 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
4049 queue_link_down = 1;
4050 dd_dev_info(dd, "Link error: %s\n",
4051 dc8051_info_err_string(buf,
4052 sizeof(buf),
4053 err & FAILED_LNI));
4054 }
4055 err &= ~(u64)FAILED_LNI;
4056 }
4057 if (err) {
4058 /* report remaining errors, but do not do anything */
4059 dd_dev_err(dd, "8051 info error: %s\n",
4060 dc8051_info_err_string(buf, sizeof(buf), err));
4061 }
4062
4063 /*
4064 * Handle host message flags.
4065 */
4066 if (host_msg & HOST_REQ_DONE) {
4067 /*
4068 * Presently, the driver does a busy wait for
4069 * host requests to complete. This is only an
4070 * informational message.
4071 * NOTE: The 8051 clears the host message
4072 * information *on the next 8051 command*.
4073 * Therefore, when linkup is achieved,
4074 * this flag will still be set.
4075 */
4076 host_msg &= ~(u64)HOST_REQ_DONE;
4077 }
4078 if (host_msg & BC_SMA_MSG) {
4079 queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
4080 host_msg &= ~(u64)BC_SMA_MSG;
4081 }
4082 if (host_msg & LINKUP_ACHIEVED) {
4083 dd_dev_info(dd, "8051: Link up\n");
4084 queue_work(ppd->hfi1_wq, &ppd->link_up_work);
4085 host_msg &= ~(u64)LINKUP_ACHIEVED;
4086 }
4087 if (host_msg & EXT_DEVICE_CFG_REQ) {
4088 handle_8051_request(dd);
4089 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
4090 }
4091 if (host_msg & VERIFY_CAP_FRAME) {
4092 queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
4093 host_msg &= ~(u64)VERIFY_CAP_FRAME;
4094 }
4095 if (host_msg & LINK_GOING_DOWN) {
4096 const char *extra = "";
4097 /* no downgrade action needed if going down */
4098 if (host_msg & LINK_WIDTH_DOWNGRADED) {
4099 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
4100 extra = " (ignoring downgrade)";
4101 }
4102 dd_dev_info(dd, "8051: Link down%s\n", extra);
4103 queue_link_down = 1;
4104 host_msg &= ~(u64)LINK_GOING_DOWN;
4105 }
4106 if (host_msg & LINK_WIDTH_DOWNGRADED) {
4107 queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
4108 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
4109 }
4110 if (host_msg) {
4111 /* report remaining messages, but do not do anything */
4112 dd_dev_info(dd, "8051 info host message: %s\n",
4113 dc8051_info_host_msg_string(buf, sizeof(buf),
4114 host_msg));
4115 }
4116
4117 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
4118 }
4119 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
4120 /*
4121 * Lost the 8051 heartbeat. If this happens, we
4122 * receive constant interrupts about it. Disable
4123 * the interrupt after the first.
4124 */
4125 dd_dev_err(dd, "Lost 8051 heartbeat\n");
4126 write_csr(dd, DC_DC8051_ERR_EN,
4127 read_csr(dd, DC_DC8051_ERR_EN)
4128 & ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
4129
4130 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
4131 }
4132 if (reg) {
4133 /* report the error, but do not do anything */
4134 dd_dev_err(dd, "8051 error: %s\n",
4135 dc8051_err_string(buf, sizeof(buf), reg));
4136 }
4137
4138 if (queue_link_down) {
4139 /* if the link is already going down or disabled, do not
4140 * queue another */
4141 if ((ppd->host_link_state
4142 & (HLS_GOING_OFFLINE|HLS_LINK_COOLDOWN))
4143 || ppd->link_enabled == 0) {
4144 dd_dev_info(dd, "%s: not queuing link down\n",
4145 __func__);
4146 } else {
4147 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
4148 }
4149 }
4150}
4151
4152static const char * const fm_config_txt[] = {
4153[0] =
4154 "BadHeadDist: Distance violation between two head flits",
4155[1] =
4156 "BadTailDist: Distance violation between two tail flits",
4157[2] =
4158 "BadCtrlDist: Distance violation between two credit control flits",
4159[3] =
4160 "BadCrdAck: Credits return for unsupported VL",
4161[4] =
4162 "UnsupportedVLMarker: Received VL Marker",
4163[5] =
4164 "BadPreempt: Exceeded the preemption nesting level",
4165[6] =
4166 "BadControlFlit: Received unsupported control flit",
4167/* no 7 */
4168[8] =
4169 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
4170};
4171
4172static const char * const port_rcv_txt[] = {
4173[1] =
4174 "BadPktLen: Illegal PktLen",
4175[2] =
4176 "PktLenTooLong: Packet longer than PktLen",
4177[3] =
4178 "PktLenTooShort: Packet shorter than PktLen",
4179[4] =
4180 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
4181[5] =
4182 "BadDLID: Illegal DLID (0, doesn't match HFI)",
4183[6] =
4184 "BadL2: Illegal L2 opcode",
4185[7] =
4186 "BadSC: Unsupported SC",
4187[9] =
4188 "BadRC: Illegal RC",
4189[11] =
4190 "PreemptError: Preempting with same VL",
4191[12] =
4192 "PreemptVL15: Preempting a VL15 packet",
4193};
4194
4195#define OPA_LDR_FMCONFIG_OFFSET 16
4196#define OPA_LDR_PORTRCV_OFFSET 0
4197static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
4198{
4199 u64 info, hdr0, hdr1;
4200 const char *extra;
4201 char buf[96];
4202 struct hfi1_pportdata *ppd = dd->pport;
4203 u8 lcl_reason = 0;
4204 int do_bounce = 0;
4205
4206 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
4207 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
4208 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
4209 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
4210 /* set status bit */
4211 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
4212 }
4213 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
4214 }
4215
4216 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
4217 struct hfi1_pportdata *ppd = dd->pport;
4218 /* this counter saturates at (2^32) - 1 */
4219 if (ppd->link_downed < (u32)UINT_MAX)
4220 ppd->link_downed++;
4221 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
4222 }
4223
4224 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
4225 u8 reason_valid = 1;
4226
4227 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
4228 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
4229 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
4230 /* set status bit */
4231 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
4232 }
4233 switch (info) {
4234 case 0:
4235 case 1:
4236 case 2:
4237 case 3:
4238 case 4:
4239 case 5:
4240 case 6:
4241 extra = fm_config_txt[info];
4242 break;
4243 case 8:
4244 extra = fm_config_txt[info];
4245 if (ppd->port_error_action &
4246 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
4247 do_bounce = 1;
4248 /*
4249 * lcl_reason cannot be derived from info
4250 * for this error
4251 */
4252 lcl_reason =
4253 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
4254 }
4255 break;
4256 default:
4257 reason_valid = 0;
4258 snprintf(buf, sizeof(buf), "reserved%lld", info);
4259 extra = buf;
4260 break;
4261 }
4262
4263 if (reason_valid && !do_bounce) {
4264 do_bounce = ppd->port_error_action &
4265 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
4266 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
4267 }
4268
4269 /* just report this */
4270 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
4271 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
4272 }
4273
4274 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
4275 u8 reason_valid = 1;
4276
4277 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
4278 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
4279 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
4280 if (!(dd->err_info_rcvport.status_and_code &
4281 OPA_EI_STATUS_SMASK)) {
4282 dd->err_info_rcvport.status_and_code =
4283 info & OPA_EI_CODE_SMASK;
4284 /* set status bit */
4285 dd->err_info_rcvport.status_and_code |=
4286 OPA_EI_STATUS_SMASK;
4287 /* save first 2 flits in the packet that caused
4288 * the error */
4289 dd->err_info_rcvport.packet_flit1 = hdr0;
4290 dd->err_info_rcvport.packet_flit2 = hdr1;
4291 }
4292 switch (info) {
4293 case 1:
4294 case 2:
4295 case 3:
4296 case 4:
4297 case 5:
4298 case 6:
4299 case 7:
4300 case 9:
4301 case 11:
4302 case 12:
4303 extra = port_rcv_txt[info];
4304 break;
4305 default:
4306 reason_valid = 0;
4307 snprintf(buf, sizeof(buf), "reserved%lld", info);
4308 extra = buf;
4309 break;
4310 }
4311
4312 if (reason_valid && !do_bounce) {
4313 do_bounce = ppd->port_error_action &
4314 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
4315 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
4316 }
4317
4318 /* just report this */
4319 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
4320 dd_dev_info(dd, " hdr0 0x%llx, hdr1 0x%llx\n",
4321 hdr0, hdr1);
4322
4323 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
4324 }
4325
4326 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
4327 /* informative only */
4328 dd_dev_info(dd, "8051 access to LCB blocked\n");
4329 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
4330 }
4331 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
4332 /* informative only */
4333 dd_dev_info(dd, "host access to LCB blocked\n");
4334 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
4335 }
4336
4337 /* report any remaining errors */
4338 if (reg)
4339 dd_dev_info(dd, "DCC Error: %s\n",
4340 dcc_err_string(buf, sizeof(buf), reg));
4341
4342 if (lcl_reason == 0)
4343 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
4344
4345 if (do_bounce) {
4346 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
4347 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
4348 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
4349 }
4350}
4351
4352static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
4353{
4354 char buf[96];
4355
4356 dd_dev_info(dd, "LCB Error: %s\n",
4357 lcb_err_string(buf, sizeof(buf), reg));
4358}
4359
4360/*
4361 * CCE block DC interrupt. Source is < 8.
4362 */
4363static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
4364{
4365 const struct err_reg_info *eri = &dc_errs[source];
4366
4367 if (eri->handler) {
4368 interrupt_clear_down(dd, 0, eri);
4369 } else if (source == 3 /* dc_lbm_int */) {
4370 /*
4371 * This indicates that a parity error has occurred on the
4372 * address/control lines presented to the LBM. The error
4373 * is a single pulse, there is no associated error flag,
4374 * and it is non-maskable. This is because if a parity
4375 * error occurs on the request the request is dropped.
4376 * This should never occur, but it is nice to know if it
4377 * ever does.
4378 */
4379 dd_dev_err(dd, "Parity error in DC LBM block\n");
4380 } else {
4381 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
4382 }
4383}
4384
4385/*
4386 * TX block send credit interrupt. Source is < 160.
4387 */
4388static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
4389{
4390 sc_group_release_update(dd, source);
4391}
4392
4393/*
4394 * TX block SDMA interrupt. Source is < 48.
4395 *
4396 * SDMA interrupts are grouped by type:
4397 *
4398 * 0 - N-1 = SDma
4399 * N - 2N-1 = SDmaProgress
4400 * 2N - 3N-1 = SDmaIdle
4401 */
4402static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
4403{
4404 /* what interrupt */
4405 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
4406 /* which engine */
4407 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
4408
4409#ifdef CONFIG_SDMA_VERBOSITY
4410 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
4411 slashstrip(__FILE__), __LINE__, __func__);
4412 sdma_dumpstate(&dd->per_sdma[which]);
4413#endif
4414
4415 if (likely(what < 3 && which < dd->num_sdma)) {
4416 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
4417 } else {
4418 /* should not happen */
4419 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
4420 }
4421}
4422
4423/*
4424 * RX block receive available interrupt. Source is < 160.
4425 */
4426static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
4427{
4428 struct hfi1_ctxtdata *rcd;
4429 char *err_detail;
4430
4431 if (likely(source < dd->num_rcv_contexts)) {
4432 rcd = dd->rcd[source];
4433 if (rcd) {
4434 if (source < dd->first_user_ctxt)
f4f30031 4435 rcd->do_interrupt(rcd, 0);
77241056
MM		 4436 else
4437 handle_user_interrupt(rcd);
4438 return; /* OK */
4439 }
4440 /* received an interrupt, but no rcd */
4441 err_detail = "dataless";
4442 } else {
4443 /* received an interrupt, but are not using that context */
4444 err_detail = "out of range";
4445 }
4446 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
4447 err_detail, source);
4448}
4449
4450/*
4451 * RX block receive urgent interrupt. Source is < 160.
4452 */
4453static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
4454{
4455 struct hfi1_ctxtdata *rcd;
4456 char *err_detail;
4457
4458 if (likely(source < dd->num_rcv_contexts)) {
4459 rcd = dd->rcd[source];
4460 if (rcd) {
4461 /* only pay attention to user urgent interrupts */
4462 if (source >= dd->first_user_ctxt)
4463 handle_user_interrupt(rcd);
4464 return; /* OK */
4465 }
4466 /* received an interrupt, but no rcd */
4467 err_detail = "dataless";
4468 } else {
4469 /* received an interrupt, but are not using that context */
4470 err_detail = "out of range";
4471 }
4472 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
4473 err_detail, source);
4474}
4475
4476/*
4477 * Reserved range interrupt. Should not be called in normal operation.
4478 */
4479static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
4480{
4481 char name[64];
4482
4483 dd_dev_err(dd, "unexpected %s interrupt\n",
4484 is_reserved_name(name, sizeof(name), source));
4485}
4486
4487static const struct is_table is_table[] = {
4488/* start end
4489 name func interrupt func */
4490{ IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
4491 is_misc_err_name, is_misc_err_int },
4492{ IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
4493 is_sdma_eng_err_name, is_sdma_eng_err_int },
4494{ IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
4495 is_sendctxt_err_name, is_sendctxt_err_int },
4496{ IS_SDMA_START, IS_SDMA_END,
4497 is_sdma_eng_name, is_sdma_eng_int },
4498{ IS_VARIOUS_START, IS_VARIOUS_END,
4499 is_various_name, is_various_int },
4500{ IS_DC_START, IS_DC_END,
4501 is_dc_name, is_dc_int },
4502{ IS_RCVAVAIL_START, IS_RCVAVAIL_END,
4503 is_rcv_avail_name, is_rcv_avail_int },
4504{ IS_RCVURGENT_START, IS_RCVURGENT_END,
4505 is_rcv_urgent_name, is_rcv_urgent_int },
4506{ IS_SENDCREDIT_START, IS_SENDCREDIT_END,
4507 is_send_credit_name, is_send_credit_int},
4508{ IS_RESERVED_START, IS_RESERVED_END,
4509 is_reserved_name, is_reserved_int},
4510};
4511
4512/*
4513 * Interrupt source interrupt - called when the given source has an interrupt.
4514 * Source is a bit index into an array of 64-bit integers.
4515 */
4516static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
4517{
4518 const struct is_table *entry;
4519
4520 /* avoids a double compare by walking the table in-order */
4521 for (entry = &is_table[0]; entry->is_name; entry++) {
4522 if (source < entry->end) {
4523 trace_hfi1_interrupt(dd, entry, source);
4524 entry->is_int(dd, source - entry->start);
4525 return;
4526 }
4527 }
4528 /* fell off the end */
4529 dd_dev_err(dd, "invalid interrupt source %u\n", source);
4530}
4531
4532/*
4533 * General interrupt handler. This is able to correctly handle
4534 * all interrupts in case INTx is used.
4535 */
4536static irqreturn_t general_interrupt(int irq, void *data)
4537{
4538 struct hfi1_devdata *dd = data;
4539 u64 regs[CCE_NUM_INT_CSRS];
4540 u32 bit;
4541 int i;
4542
4543 this_cpu_inc(*dd->int_counter);
4544
4545 /* phase 1: scan and clear all handled interrupts */
4546 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
4547 if (dd->gi_mask[i] == 0) {
4548 regs[i] = 0; /* used later */
4549 continue;
4550 }
4551 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
4552 dd->gi_mask[i];
4553 /* only clear if anything is set */
4554 if (regs[i])
4555 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
4556 }
4557
4558 /* phase 2: call the appropriate handler */
4559 for_each_set_bit(bit, (unsigned long *)&regs[0],
4560 CCE_NUM_INT_CSRS*64) {
4561 is_interrupt(dd, bit);
4562 }
4563
4564 return IRQ_HANDLED;
4565}
4566
4567static irqreturn_t sdma_interrupt(int irq, void *data)
4568{
4569 struct sdma_engine *sde = data;
4570 struct hfi1_devdata *dd = sde->dd;
4571 u64 status;
4572
4573#ifdef CONFIG_SDMA_VERBOSITY
4574 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
4575 slashstrip(__FILE__), __LINE__, __func__);
4576 sdma_dumpstate(sde);
4577#endif
4578
4579 this_cpu_inc(*dd->int_counter);
4580
4581 /* This read_csr is really bad in the hot path */
4582 status = read_csr(dd,
4583 CCE_INT_STATUS + (8*(IS_SDMA_START/64)))
4584 & sde->imask;
4585 if (likely(status)) {
4586 /* clear the interrupt(s) */
4587 write_csr(dd,
4588 CCE_INT_CLEAR + (8*(IS_SDMA_START/64)),
4589 status);
4590
4591 /* handle the interrupt(s) */
4592 sdma_engine_interrupt(sde, status);
4593 } else
4594 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
4595 sde->this_idx);
4596
4597 return IRQ_HANDLED;
4598}
4599
4600/*
f4f30031
DL		 4601 * Clear the receive interrupt, forcing the write and making sure
4602 * we have data from the chip, pushing everything in front of it
4603 * back to the host.
4604 */
4605static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
4606{
4607 struct hfi1_devdata *dd = rcd->dd;
4608 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
4609
4610 mmiowb(); /* make sure everything before is written */
4611 write_csr(dd, addr, rcd->imask);
4612 /* force the above write on the chip and get a value back */
4613 (void)read_csr(dd, addr);
4614}
4615
4616/* force the receive interrupt */
4617static inline void force_recv_intr(struct hfi1_ctxtdata *rcd)
4618{
4619 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
4620}
4621
4622/* return non-zero if a packet is present */
4623static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
4624{
4625 if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
4626 return (rcd->seq_cnt ==
4627 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
4628
4629 /* else is RDMA rtail */
4630 return (rcd->head != get_rcvhdrtail(rcd));
4631}
4632
4633/*
4634 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
4635 * This routine will try to handle packets immediately (latency), but if
4636 * it finds too many, it will invoke the thread handler (bandwitdh). The
4637 * chip receive interupt is *not* cleared down until this or the thread (if
4638 * invoked) is finished. The intent is to avoid extra interrupts while we
4639 * are processing packets anyway.
77241056
MM		 4640 */
4641static irqreturn_t receive_context_interrupt(int irq, void *data)
4642{
4643 struct hfi1_ctxtdata *rcd = data;
4644 struct hfi1_devdata *dd = rcd->dd;
f4f30031
DL		 4645 int disposition;
4646 int present;
77241056
MM		 4647
4648 trace_hfi1_receive_interrupt(dd, rcd->ctxt);
4649 this_cpu_inc(*dd->int_counter);
4650
f4f30031
DL		 4651 /* receive interrupt remains blocked while processing packets */
4652 disposition = rcd->do_interrupt(rcd, 0);
77241056 4653
f4f30031
DL		 4654 /*
4655 * Too many packets were seen while processing packets in this
4656 * IRQ handler. Invoke the handler thread. The receive interrupt
4657 * remains blocked.
4658 */
4659 if (disposition == RCV_PKT_LIMIT)
4660 return IRQ_WAKE_THREAD;
4661
4662 /*
4663 * The packet processor detected no more packets. Clear the receive
4664 * interrupt and recheck for a packet packet that may have arrived
4665 * after the previous check and interrupt clear. If a packet arrived,
4666 * force another interrupt.
4667 */
4668 clear_recv_intr(rcd);
4669 present = check_packet_present(rcd);
4670 if (present)
4671 force_recv_intr(rcd);
4672
4673 return IRQ_HANDLED;
4674}
4675
4676/*
4677 * Receive packet thread handler. This expects to be invoked with the
4678 * receive interrupt still blocked.
4679 */
4680static irqreturn_t receive_context_thread(int irq, void *data)
4681{
4682 struct hfi1_ctxtdata *rcd = data;
4683 int present;
4684
4685 /* receive interrupt is still blocked from the IRQ handler */
4686 (void)rcd->do_interrupt(rcd, 1);
4687
4688 /*
4689 * The packet processor will only return if it detected no more
4690 * packets. Hold IRQs here so we can safely clear the interrupt and
4691 * recheck for a packet that may have arrived after the previous
4692 * check and the interrupt clear. If a packet arrived, force another
4693 * interrupt.
4694 */
4695 local_irq_disable();
4696 clear_recv_intr(rcd);
4697 present = check_packet_present(rcd);
4698 if (present)
4699 force_recv_intr(rcd);
4700 local_irq_enable();
77241056
MM		 4701
4702 return IRQ_HANDLED;
4703}
4704
4705/* ========================================================================= */
4706
4707u32 read_physical_state(struct hfi1_devdata *dd)
4708{
4709 u64 reg;
4710
4711 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
4712 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
4713 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
4714}
4715
4716static u32 read_logical_state(struct hfi1_devdata *dd)
4717{
4718 u64 reg;
4719
4720 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
4721 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
4722 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
4723}
4724
4725static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
4726{
4727 u64 reg;
4728
4729 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
4730 /* clear current state, set new state */
4731 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
4732 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
4733 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
4734}
4735
4736/*
4737 * Use the 8051 to read a LCB CSR.
4738 */
4739static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
4740{
4741 u32 regno;
4742 int ret;
4743
4744 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
4745 if (acquire_lcb_access(dd, 0) == 0) {
4746 *data = read_csr(dd, addr);
4747 release_lcb_access(dd, 0);
4748 return 0;
4749 }
4750 return -EBUSY;
4751 }
4752
4753 /* register is an index of LCB registers: (offset - base) / 8 */
4754 regno = (addr - DC_LCB_CFG_RUN) >> 3;
4755 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
4756 if (ret != HCMD_SUCCESS)
4757 return -EBUSY;
4758 return 0;
4759}
4760
4761/*
4762 * Read an LCB CSR. Access may not be in host control, so check.
4763 * Return 0 on success, -EBUSY on failure.
4764 */
4765int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
4766{
4767 struct hfi1_pportdata *ppd = dd->pport;
4768
4769 /* if up, go through the 8051 for the value */
4770 if (ppd->host_link_state & HLS_UP)
4771 return read_lcb_via_8051(dd, addr, data);
4772 /* if going up or down, no access */
4773 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
4774 return -EBUSY;
4775 /* otherwise, host has access */
4776 *data = read_csr(dd, addr);
4777 return 0;
4778}
4779
4780/*
4781 * Use the 8051 to write a LCB CSR.
4782 */
4783static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
4784{
3bf40d65
DL		 4785 u32 regno;
4786 int ret;
77241056 4787
3bf40d65
DL		 4788 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
4789 (dd->dc8051_ver < dc8051_ver(0, 20))) {
4790 if (acquire_lcb_access(dd, 0) == 0) {
4791 write_csr(dd, addr, data);
4792 release_lcb_access(dd, 0);
4793 return 0;
4794 }
4795 return -EBUSY;
77241056 4796 }
3bf40d65
DL		 4797
4798 /* register is an index of LCB registers: (offset - base) / 8 */
4799 regno = (addr - DC_LCB_CFG_RUN) >> 3;
4800 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
4801 if (ret != HCMD_SUCCESS)
4802 return -EBUSY;
4803 return 0;
77241056
MM		 4804}
4805
4806/*
4807 * Write an LCB CSR. Access may not be in host control, so check.
4808 * Return 0 on success, -EBUSY on failure.
4809 */
4810int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
4811{
4812 struct hfi1_pportdata *ppd = dd->pport;
4813
4814 /* if up, go through the 8051 for the value */
4815 if (ppd->host_link_state & HLS_UP)
4816 return write_lcb_via_8051(dd, addr, data);
4817 /* if going up or down, no access */
4818 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
4819 return -EBUSY;
4820 /* otherwise, host has access */
4821 write_csr(dd, addr, data);
4822 return 0;
4823}
4824
4825/*
4826 * Returns:
4827 * < 0 = Linux error, not able to get access
4828 * > 0 = 8051 command RETURN_CODE
4829 */
4830static int do_8051_command(
4831 struct hfi1_devdata *dd,
4832 u32 type,
4833 u64 in_data,
4834 u64 *out_data)
4835{
4836 u64 reg, completed;
4837 int return_code;
4838 unsigned long flags;
4839 unsigned long timeout;
4840
4841 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
4842
4843 /*
4844 * Alternative to holding the lock for a long time:
4845 * - keep busy wait - have other users bounce off
4846 */
4847 spin_lock_irqsave(&dd->dc8051_lock, flags);
4848
4849 /* We can't send any commands to the 8051 if it's in reset */
4850 if (dd->dc_shutdown) {
4851 return_code = -ENODEV;
4852 goto fail;
4853 }
4854
4855 /*
4856 * If an 8051 host command timed out previously, then the 8051 is
4857 * stuck.
4858 *
4859 * On first timeout, attempt to reset and restart the entire DC
4860 * block (including 8051). (Is this too big of a hammer?)
4861 *
4862 * If the 8051 times out a second time, the reset did not bring it
4863 * back to healthy life. In that case, fail any subsequent commands.
4864 */
4865 if (dd->dc8051_timed_out) {
4866 if (dd->dc8051_timed_out > 1) {
4867 dd_dev_err(dd,
4868 "Previous 8051 host command timed out, skipping command %u\n",
4869 type);
4870 return_code = -ENXIO;
4871 goto fail;
4872 }
4873 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
4874 dc_shutdown(dd);
4875 dc_start(dd);
4876 spin_lock_irqsave(&dd->dc8051_lock, flags);
4877 }
4878
4879 /*
4880 * If there is no timeout, then the 8051 command interface is
4881 * waiting for a command.
4882 */
4883
3bf40d65
DL		 4884 /*
4885 * When writing a LCB CSR, out_data contains the full value to
4886 * to be written, while in_data contains the relative LCB
4887 * address in 7:0. Do the work here, rather than the caller,
4888 * of distrubting the write data to where it needs to go:
4889 *
4890 * Write data
4891 * 39:00 -> in_data[47:8]
4892 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
4893 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
4894 */
4895 if (type == HCMD_WRITE_LCB_CSR) {
4896 in_data |= ((*out_data) & 0xffffffffffull) << 8;
4897 reg = ((((*out_data) >> 40) & 0xff) <<
4898 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
4899 | ((((*out_data) >> 48) & 0xffff) <<
4900 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
4901 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
4902 }
4903
77241056
MM		 4904 /*
4905 * Do two writes: the first to stabilize the type and req_data, the
4906 * second to activate.
4907 */
4908 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
4909 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
4910 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
4911 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
4912 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
4913 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
4914 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
4915
4916 /* wait for completion, alternate: interrupt */
4917 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
4918 while (1) {
4919 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
4920 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
4921 if (completed)
4922 break;
4923 if (time_after(jiffies, timeout)) {
4924 dd->dc8051_timed_out++;
4925 dd_dev_err(dd, "8051 host command %u timeout\n", type);
4926 if (out_data)
4927 *out_data = 0;
4928 return_code = -ETIMEDOUT;
4929 goto fail;
4930 }
4931 udelay(2);
4932 }
4933
4934 if (out_data) {
4935 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
4936 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
4937 if (type == HCMD_READ_LCB_CSR) {
4938 /* top 16 bits are in a different register */
4939 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
4940 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
4941 << (48
4942 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
4943 }
4944 }
4945 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
4946 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
4947 dd->dc8051_timed_out = 0;
4948 /*
4949 * Clear command for next user.
4950 */
4951 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
4952
4953fail:
4954 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
4955
4956 return return_code;
4957}
4958
4959static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
4960{
4961 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
4962}
4963
4964static int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
4965 u8 lane_id, u32 config_data)
4966{
4967 u64 data;
4968 int ret;
4969
4970 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
4971 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
4972 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
4973 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
4974 if (ret != HCMD_SUCCESS) {
4975 dd_dev_err(dd,
4976 "load 8051 config: field id %d, lane %d, err %d\n",
4977 (int)field_id, (int)lane_id, ret);
4978 }
4979 return ret;
4980}
4981
4982/*
4983 * Read the 8051 firmware "registers". Use the RAM directly. Always
4984 * set the result, even on error.
4985 * Return 0 on success, -errno on failure
4986 */
4987static int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
4988 u32 *result)
4989{
4990 u64 big_data;
4991 u32 addr;
4992 int ret;
4993
4994 /* address start depends on the lane_id */
4995 if (lane_id < 4)
4996 addr = (4 * NUM_GENERAL_FIELDS)
4997 + (lane_id * 4 * NUM_LANE_FIELDS);
4998 else
4999 addr = 0;
5000 addr += field_id * 4;
5001
5002 /* read is in 8-byte chunks, hardware will truncate the address down */
5003 ret = read_8051_data(dd, addr, 8, &big_data);
5004
5005 if (ret == 0) {
5006 /* extract the 4 bytes we want */
5007 if (addr & 0x4)
5008 *result = (u32)(big_data >> 32);
5009 else
5010 *result = (u32)big_data;
5011 } else {
5012 *result = 0;
5013 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
5014 __func__, lane_id, field_id);
5015 }
5016
5017 return ret;
5018}
5019
5020static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
5021 u8 continuous)
5022{
5023 u32 frame;
5024
5025 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
5026 | power_management << POWER_MANAGEMENT_SHIFT;
5027 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
5028 GENERAL_CONFIG, frame);
5029}
5030
5031static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
5032 u16 vl15buf, u8 crc_sizes)
5033{
5034 u32 frame;
5035
5036 frame = (u32)vau << VAU_SHIFT
5037 | (u32)z << Z_SHIFT
5038 | (u32)vcu << VCU_SHIFT
5039 | (u32)vl15buf << VL15BUF_SHIFT
5040 | (u32)crc_sizes << CRC_SIZES_SHIFT;
5041 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
5042 GENERAL_CONFIG, frame);
5043}
5044
5045static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
5046 u8 *flag_bits, u16 *link_widths)
5047{
5048 u32 frame;
5049
5050 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
5051 &frame);
5052 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
5053 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
5054 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
5055}
5056
5057static int write_vc_local_link_width(struct hfi1_devdata *dd,
5058 u8 misc_bits,
5059 u8 flag_bits,
5060 u16 link_widths)
5061{
5062 u32 frame;
5063
5064 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
5065 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
5066 | (u32)link_widths << LINK_WIDTH_SHIFT;
5067 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
5068 frame);
5069}
5070
5071static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
5072 u8 device_rev)
5073{
5074 u32 frame;
5075
5076 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
5077 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
5078 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
5079}
5080
5081static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
5082 u8 *device_rev)
5083{
5084 u32 frame;
5085
5086 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
5087 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
5088 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
5089 & REMOTE_DEVICE_REV_MASK;
5090}
5091
5092void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
5093{
5094 u32 frame;
5095
5096 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
5097 *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
5098 *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
5099}
5100
5101static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
5102 u8 *continuous)
5103{
5104 u32 frame;
5105
5106 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
5107 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
5108 & POWER_MANAGEMENT_MASK;
5109 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
5110 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
5111}
5112
5113static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
5114 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
5115{
5116 u32 frame;
5117
5118 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
5119 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
5120 *z = (frame >> Z_SHIFT) & Z_MASK;
5121 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
5122 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
5123 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
5124}
5125
5126static void read_vc_remote_link_width(struct hfi1_devdata *dd,
5127 u8 *remote_tx_rate,
5128 u16 *link_widths)
5129{
5130 u32 frame;
5131
5132 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
5133 &frame);
5134 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
5135 & REMOTE_TX_RATE_MASK;
5136 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
5137}
5138
5139static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
5140{
5141 u32 frame;
5142
5143 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
5144 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
5145}
5146
5147static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
5148{
5149 u32 frame;
5150
5151 read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
5152 *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
5153}
5154
5155static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
5156{
5157 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
5158}
5159
5160static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
5161{
5162 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
5163}
5164
5165void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
5166{
5167 u32 frame;
5168 int ret;
5169
5170 *link_quality = 0;
5171 if (dd->pport->host_link_state & HLS_UP) {
5172 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
5173 &frame);
5174 if (ret == 0)
5175 *link_quality = (frame >> LINK_QUALITY_SHIFT)
5176 & LINK_QUALITY_MASK;
5177 }
5178}
5179
5180static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
5181{
5182 u32 frame;
5183
5184 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
5185 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
5186}
5187
5188static int read_tx_settings(struct hfi1_devdata *dd,
5189 u8 *enable_lane_tx,
5190 u8 *tx_polarity_inversion,
5191 u8 *rx_polarity_inversion,
5192 u8 *max_rate)
5193{
5194 u32 frame;
5195 int ret;
5196
5197 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
5198 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
5199 & ENABLE_LANE_TX_MASK;
5200 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
5201 & TX_POLARITY_INVERSION_MASK;
5202 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
5203 & RX_POLARITY_INVERSION_MASK;
5204 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
5205 return ret;
5206}
5207
5208static int write_tx_settings(struct hfi1_devdata *dd,
5209 u8 enable_lane_tx,
5210 u8 tx_polarity_inversion,
5211 u8 rx_polarity_inversion,
5212 u8 max_rate)
5213{
5214 u32 frame;
5215
5216 /* no need to mask, all variable sizes match field widths */
5217 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
5218 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
5219 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
5220 | max_rate << MAX_RATE_SHIFT;
5221 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
5222}
5223
5224static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
5225{
5226 u32 frame, version, prod_id;
5227 int ret, lane;
5228
5229 /* 4 lanes */
5230 for (lane = 0; lane < 4; lane++) {
5231 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
5232 if (ret) {
5233 dd_dev_err(
5234 dd,
5235 "Unable to read lane %d firmware details\n",
5236 lane);
5237 continue;
5238 }
5239 version = (frame >> SPICO_ROM_VERSION_SHIFT)
5240 & SPICO_ROM_VERSION_MASK;
5241 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
5242 & SPICO_ROM_PROD_ID_MASK;
5243 dd_dev_info(dd,
5244 "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
5245 lane, version, prod_id);
5246 }
5247}
5248
5249/*
5250 * Read an idle LCB message.
5251 *
5252 * Returns 0 on success, -EINVAL on error
5253 */
5254static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
5255{
5256 int ret;
5257
5258 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG,
5259 type, data_out);
5260 if (ret != HCMD_SUCCESS) {
5261 dd_dev_err(dd, "read idle message: type %d, err %d\n",
5262 (u32)type, ret);
5263 return -EINVAL;
5264 }
5265 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
5266 /* return only the payload as we already know the type */
5267 *data_out >>= IDLE_PAYLOAD_SHIFT;
5268 return 0;
5269}
5270
5271/*
5272 * Read an idle SMA message. To be done in response to a notification from
5273 * the 8051.
5274 *
5275 * Returns 0 on success, -EINVAL on error
5276 */
5277static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
5278{
5279 return read_idle_message(dd,
5280 (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, data);
5281}
5282
5283/*
5284 * Send an idle LCB message.
5285 *
5286 * Returns 0 on success, -EINVAL on error
5287 */
5288static int send_idle_message(struct hfi1_devdata *dd, u64 data)
5289{
5290 int ret;
5291
5292 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
5293 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
5294 if (ret != HCMD_SUCCESS) {
5295 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
5296 data, ret);
5297 return -EINVAL;
5298 }
5299 return 0;
5300}
5301
5302/*
5303 * Send an idle SMA message.
5304 *
5305 * Returns 0 on success, -EINVAL on error
5306 */
5307int send_idle_sma(struct hfi1_devdata *dd, u64 message)
5308{
5309 u64 data;
5310
5311 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT)
5312 | ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
5313 return send_idle_message(dd, data);
5314}
5315
5316/*
5317 * Initialize the LCB then do a quick link up. This may or may not be
5318 * in loopback.
5319 *
5320 * return 0 on success, -errno on error
5321 */
5322static int do_quick_linkup(struct hfi1_devdata *dd)
5323{
5324 u64 reg;
5325 unsigned long timeout;
5326 int ret;
5327
5328 lcb_shutdown(dd, 0);
5329
5330 if (loopback) {
5331 /* LCB_CFG_LOOPBACK.VAL = 2 */
5332 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
5333 write_csr(dd, DC_LCB_CFG_LOOPBACK,
5334 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
5335 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
5336 }
5337
5338 /* start the LCBs */
5339 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
5340 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
5341
5342 /* simulator only loopback steps */
5343 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
5344 /* LCB_CFG_RUN.EN = 1 */
5345 write_csr(dd, DC_LCB_CFG_RUN,
5346 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
5347
5348 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
5349 timeout = jiffies + msecs_to_jiffies(10);
5350 while (1) {
5351 reg = read_csr(dd,
5352 DC_LCB_STS_LINK_TRANSFER_ACTIVE);
5353 if (reg)
5354 break;
5355 if (time_after(jiffies, timeout)) {
5356 dd_dev_err(dd,
5357 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
5358 return -ETIMEDOUT;
5359 }
5360 udelay(2);
5361 }
5362
5363 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
5364 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
5365 }
5366
5367 if (!loopback) {
5368 /*
5369 * When doing quick linkup and not in loopback, both
5370 * sides must be done with LCB set-up before either
5371 * starts the quick linkup. Put a delay here so that
5372 * both sides can be started and have a chance to be
5373 * done with LCB set up before resuming.
5374 */
5375 dd_dev_err(dd,
5376 "Pausing for peer to be finished with LCB set up\n");
5377 msleep(5000);
5378 dd_dev_err(dd,
5379 "Continuing with quick linkup\n");
5380 }
5381
5382 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
5383 set_8051_lcb_access(dd);
5384
5385 /*
5386 * State "quick" LinkUp request sets the physical link state to
5387 * LinkUp without a verify capability sequence.
5388 * This state is in simulator v37 and later.
5389 */
5390 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
5391 if (ret != HCMD_SUCCESS) {
5392 dd_dev_err(dd,
5393 "%s: set physical link state to quick LinkUp failed with return %d\n",
5394 __func__, ret);
5395
5396 set_host_lcb_access(dd);
5397 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
5398
5399 if (ret >= 0)
5400 ret = -EINVAL;
5401 return ret;
5402 }
5403
5404 return 0; /* success */
5405}
5406
5407/*
5408 * Set the SerDes to internal loopback mode.
5409 * Returns 0 on success, -errno on error.
5410 */
5411static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
5412{
5413 int ret;
5414
5415 ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
5416 if (ret == HCMD_SUCCESS)
5417 return 0;
5418 dd_dev_err(dd,
5419 "Set physical link state to SerDes Loopback failed with return %d\n",
5420 ret);
5421 if (ret >= 0)
5422 ret = -EINVAL;
5423 return ret;
5424}
5425
5426/*
5427 * Do all special steps to set up loopback.
5428 */
5429static int init_loopback(struct hfi1_devdata *dd)
5430{
5431 dd_dev_info(dd, "Entering loopback mode\n");
5432
5433 /* all loopbacks should disable self GUID check */
5434 write_csr(dd, DC_DC8051_CFG_MODE,
5435 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
5436
5437 /*
5438 * The simulator has only one loopback option - LCB. Switch
5439 * to that option, which includes quick link up.
5440 *
5441 * Accept all valid loopback values.
5442 */
5443 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
5444 && (loopback == LOOPBACK_SERDES
5445 || loopback == LOOPBACK_LCB
5446 || loopback == LOOPBACK_CABLE)) {
5447 loopback = LOOPBACK_LCB;
5448 quick_linkup = 1;
5449 return 0;
5450 }
5451
5452 /* handle serdes loopback */
5453 if (loopback == LOOPBACK_SERDES) {
5454 /* internal serdes loopack needs quick linkup on RTL */
5455 if (dd->icode == ICODE_RTL_SILICON)
5456 quick_linkup = 1;
5457 return set_serdes_loopback_mode(dd);
5458 }
5459
5460 /* LCB loopback - handled at poll time */
5461 if (loopback == LOOPBACK_LCB) {
5462 quick_linkup = 1; /* LCB is always quick linkup */
5463
5464 /* not supported in emulation due to emulation RTL changes */
5465 if (dd->icode == ICODE_FPGA_EMULATION) {
5466 dd_dev_err(dd,
5467 "LCB loopback not supported in emulation\n");
5468 return -EINVAL;
5469 }
5470 return 0;
5471 }
5472
5473 /* external cable loopback requires no extra steps */
5474 if (loopback == LOOPBACK_CABLE)
5475 return 0;
5476
5477 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
5478 return -EINVAL;
5479}
5480
5481/*
5482 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
5483 * used in the Verify Capability link width attribute.
5484 */
5485static u16 opa_to_vc_link_widths(u16 opa_widths)
5486{
5487 int i;
5488 u16 result = 0;
5489
5490 static const struct link_bits {
5491 u16 from;
5492 u16 to;
5493 } opa_link_xlate[] = {
5494 { OPA_LINK_WIDTH_1X, 1 << (1-1) },
5495 { OPA_LINK_WIDTH_2X, 1 << (2-1) },
5496 { OPA_LINK_WIDTH_3X, 1 << (3-1) },
5497 { OPA_LINK_WIDTH_4X, 1 << (4-1) },
5498 };
5499
5500 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
5501 if (opa_widths & opa_link_xlate[i].from)
5502 result |= opa_link_xlate[i].to;
5503 }
5504 return result;
5505}
5506
5507/*
5508 * Set link attributes before moving to polling.
5509 */
5510static int set_local_link_attributes(struct hfi1_pportdata *ppd)
5511{
5512 struct hfi1_devdata *dd = ppd->dd;
5513 u8 enable_lane_tx;
5514 u8 tx_polarity_inversion;
5515 u8 rx_polarity_inversion;
5516 int ret;
5517
5518 /* reset our fabric serdes to clear any lingering problems */
5519 fabric_serdes_reset(dd);
5520
5521 /* set the local tx rate - need to read-modify-write */
5522 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
5523 &rx_polarity_inversion, &ppd->local_tx_rate);
5524 if (ret)
5525 goto set_local_link_attributes_fail;
5526
5527 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
5528 /* set the tx rate to the fastest enabled */
5529 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
5530 ppd->local_tx_rate = 1;
5531 else
5532 ppd->local_tx_rate = 0;
5533 } else {
5534 /* set the tx rate to all enabled */
5535 ppd->local_tx_rate = 0;
5536 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
5537 ppd->local_tx_rate |= 2;
5538 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
5539 ppd->local_tx_rate |= 1;
5540 }
febffe2c
EH		 5541
5542 enable_lane_tx = 0xF; /* enable all four lanes */
77241056
MM		 5543 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
5544 rx_polarity_inversion, ppd->local_tx_rate);
5545 if (ret != HCMD_SUCCESS)
5546 goto set_local_link_attributes_fail;
5547
5548 /*
5549 * DC supports continuous updates.
5550 */
5551 ret = write_vc_local_phy(dd, 0 /* no power management */,
5552 1 /* continuous updates */);
5553 if (ret != HCMD_SUCCESS)
5554 goto set_local_link_attributes_fail;
5555
5556 /* z=1 in the next call: AU of 0 is not supported by the hardware */
5557 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
5558 ppd->port_crc_mode_enabled);
5559 if (ret != HCMD_SUCCESS)
5560 goto set_local_link_attributes_fail;
5561
5562 ret = write_vc_local_link_width(dd, 0, 0,
5563 opa_to_vc_link_widths(ppd->link_width_enabled));
5564 if (ret != HCMD_SUCCESS)
5565 goto set_local_link_attributes_fail;
5566
5567 /* let peer know who we are */
5568 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
5569 if (ret == HCMD_SUCCESS)
5570 return 0;
5571
5572set_local_link_attributes_fail:
5573 dd_dev_err(dd,
5574 "Failed to set local link attributes, return 0x%x\n",
5575 ret);
5576 return ret;
5577}
5578
5579/*
5580 * Call this to start the link. Schedule a retry if the cable is not
5581 * present or if unable to start polling. Do not do anything if the
5582 * link is disabled. Returns 0 if link is disabled or moved to polling
5583 */
5584int start_link(struct hfi1_pportdata *ppd)
5585{
5586 if (!ppd->link_enabled) {
5587 dd_dev_info(ppd->dd,
5588 "%s: stopping link start because link is disabled\n",
5589 __func__);
5590 return 0;
5591 }
5592 if (!ppd->driver_link_ready) {
5593 dd_dev_info(ppd->dd,
5594 "%s: stopping link start because driver is not ready\n",
5595 __func__);
5596 return 0;
5597 }
5598
5599 if (qsfp_mod_present(ppd) || loopback == LOOPBACK_SERDES ||
5600 loopback == LOOPBACK_LCB ||
5601 ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
5602 return set_link_state(ppd, HLS_DN_POLL);
5603
5604 dd_dev_info(ppd->dd,
5605 "%s: stopping link start because no cable is present\n",
5606 __func__);
5607 return -EAGAIN;
5608}
5609
5610static void reset_qsfp(struct hfi1_pportdata *ppd)
5611{
5612 struct hfi1_devdata *dd = ppd->dd;
5613 u64 mask, qsfp_mask;
5614
5615 mask = (u64)QSFP_HFI0_RESET_N;
5616 qsfp_mask = read_csr(dd,
5617 dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE);
5618 qsfp_mask |= mask;
5619 write_csr(dd,
5620 dd->hfi1_id ? ASIC_QSFP2_OE : ASIC_QSFP1_OE,
5621 qsfp_mask);
5622
5623 qsfp_mask = read_csr(dd,
5624 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
5625 qsfp_mask &= ~mask;
5626 write_csr(dd,
5627 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT,
5628 qsfp_mask);
5629
5630 udelay(10);
5631
5632 qsfp_mask |= mask;
5633 write_csr(dd,
5634 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT,
5635 qsfp_mask);
5636}
5637
5638static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
5639 u8 *qsfp_interrupt_status)
5640{
5641 struct hfi1_devdata *dd = ppd->dd;
5642
5643 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
5644 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
5645 dd_dev_info(dd,
5646 "%s: QSFP cable on fire\n",
5647 __func__);
5648
5649 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
5650 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
5651 dd_dev_info(dd,
5652 "%s: QSFP cable temperature too low\n",
5653 __func__);
5654
5655 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
5656 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
5657 dd_dev_info(dd,
5658 "%s: QSFP supply voltage too high\n",
5659 __func__);
5660
5661 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
5662 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
5663 dd_dev_info(dd,
5664 "%s: QSFP supply voltage too low\n",
5665 __func__);
5666
5667 /* Byte 2 is vendor specific */
5668
5669 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
5670 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
5671 dd_dev_info(dd,
5672 "%s: Cable RX channel 1/2 power too high\n",
5673 __func__);
5674
5675 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
5676 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
5677 dd_dev_info(dd,
5678 "%s: Cable RX channel 1/2 power too low\n",
5679 __func__);
5680
5681 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
5682 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
5683 dd_dev_info(dd,
5684 "%s: Cable RX channel 3/4 power too high\n",
5685 __func__);
5686
5687 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
5688 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
5689 dd_dev_info(dd,
5690 "%s: Cable RX channel 3/4 power too low\n",
5691 __func__);
5692
5693 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
5694 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
5695 dd_dev_info(dd,
5696 "%s: Cable TX channel 1/2 bias too high\n",
5697 __func__);
5698
5699 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
5700 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
5701 dd_dev_info(dd,
5702 "%s: Cable TX channel 1/2 bias too low\n",
5703 __func__);
5704
5705 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
5706 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
5707 dd_dev_info(dd,
5708 "%s: Cable TX channel 3/4 bias too high\n",
5709 __func__);
5710
5711 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
5712 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
5713 dd_dev_info(dd,
5714 "%s: Cable TX channel 3/4 bias too low\n",
5715 __func__);
5716
5717 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
5718 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
5719 dd_dev_info(dd,
5720 "%s: Cable TX channel 1/2 power too high\n",
5721 __func__);
5722
5723 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
5724 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
5725 dd_dev_info(dd,
5726 "%s: Cable TX channel 1/2 power too low\n",
5727 __func__);
5728
5729 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
5730 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
5731 dd_dev_info(dd,
5732 "%s: Cable TX channel 3/4 power too high\n",
5733 __func__);
5734
5735 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
5736 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
5737 dd_dev_info(dd,
5738 "%s: Cable TX channel 3/4 power too low\n",
5739 __func__);
5740
5741 /* Bytes 9-10 and 11-12 are reserved */
5742 /* Bytes 13-15 are vendor specific */
5743
5744 return 0;
5745}
5746
5747static int do_pre_lni_host_behaviors(struct hfi1_pportdata *ppd)
5748{
5749 refresh_qsfp_cache(ppd, &ppd->qsfp_info);
5750
5751 return 0;
5752}
5753
5754static int do_qsfp_intr_fallback(struct hfi1_pportdata *ppd)
5755{
5756 struct hfi1_devdata *dd = ppd->dd;
5757 u8 qsfp_interrupt_status = 0;
5758
5759 if (qsfp_read(ppd, dd->hfi1_id, 2, &qsfp_interrupt_status, 1)
5760 != 1) {
5761 dd_dev_info(dd,
5762 "%s: Failed to read status of QSFP module\n",
5763 __func__);
5764 return -EIO;
5765 }
5766
5767 /* We don't care about alarms & warnings with a non-functional INT_N */
5768 if (!(qsfp_interrupt_status & QSFP_DATA_NOT_READY))
5769 do_pre_lni_host_behaviors(ppd);
5770
5771 return 0;
5772}
5773
5774/* This routine will only be scheduled if the QSFP module is present */
5775static void qsfp_event(struct work_struct *work)
5776{
5777 struct qsfp_data *qd;
5778 struct hfi1_pportdata *ppd;
5779 struct hfi1_devdata *dd;
5780
5781 qd = container_of(work, struct qsfp_data, qsfp_work);
5782 ppd = qd->ppd;
5783 dd = ppd->dd;
5784
5785 /* Sanity check */
5786 if (!qsfp_mod_present(ppd))
5787 return;
5788
5789 /*
5790 * Turn DC back on after cables has been
5791 * re-inserted. Up until now, the DC has been in
5792 * reset to save power.
5793 */
5794 dc_start(dd);
5795
5796 if (qd->cache_refresh_required) {
5797 msleep(3000);
5798 reset_qsfp(ppd);
5799
5800 /* Check for QSFP interrupt after t_init (SFF 8679)
5801 * + extra
5802 */
5803 msleep(3000);
5804 if (!qd->qsfp_interrupt_functional) {
5805 if (do_qsfp_intr_fallback(ppd) < 0)
5806 dd_dev_info(dd, "%s: QSFP fallback failed\n",
5807 __func__);
5808 ppd->driver_link_ready = 1;
5809 start_link(ppd);
5810 }
5811 }
5812
5813 if (qd->check_interrupt_flags) {
5814 u8 qsfp_interrupt_status[16] = {0,};
5815
5816 if (qsfp_read(ppd, dd->hfi1_id, 6,
5817 &qsfp_interrupt_status[0], 16) != 16) {
5818 dd_dev_info(dd,
5819 "%s: Failed to read status of QSFP module\n",
5820 __func__);
5821 } else {
5822 unsigned long flags;
5823 u8 data_status;
5824
5825 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5826 ppd->qsfp_info.check_interrupt_flags = 0;
5827 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
5828 flags);
5829
5830 if (qsfp_read(ppd, dd->hfi1_id, 2, &data_status, 1)
5831 != 1) {
5832 dd_dev_info(dd,
5833 "%s: Failed to read status of QSFP module\n",
5834 __func__);
5835 }
5836 if (!(data_status & QSFP_DATA_NOT_READY)) {
5837 do_pre_lni_host_behaviors(ppd);
5838 start_link(ppd);
5839 } else
5840 handle_qsfp_error_conditions(ppd,
5841 qsfp_interrupt_status);
5842 }
5843 }
5844}
5845
5846void init_qsfp(struct hfi1_pportdata *ppd)
5847{
5848 struct hfi1_devdata *dd = ppd->dd;
5849 u64 qsfp_mask;
5850
5851 if (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
3c2f85b8 5852 ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
77241056
MM		 5853 ppd->driver_link_ready = 1;
5854 return;
5855 }
5856
5857 ppd->qsfp_info.ppd = ppd;
5858 INIT_WORK(&ppd->qsfp_info.qsfp_work, qsfp_event);
5859
5860 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5861 /* Clear current status to avoid spurious interrupts */
5862 write_csr(dd,
5863 dd->hfi1_id ?
5864 ASIC_QSFP2_CLEAR :
5865 ASIC_QSFP1_CLEAR,
5866 qsfp_mask);
5867
5868 /* Handle active low nature of INT_N and MODPRST_N pins */
5869 if (qsfp_mod_present(ppd))
5870 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
5871 write_csr(dd,
5872 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
5873 qsfp_mask);
5874
5875 /* Allow only INT_N and MODPRST_N to trigger QSFP interrupts */
5876 qsfp_mask |= (u64)QSFP_HFI0_MODPRST_N;
5877 write_csr(dd,
5878 dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
5879 qsfp_mask);
5880
5881 if (qsfp_mod_present(ppd)) {
5882 msleep(3000);
5883 reset_qsfp(ppd);
5884
5885 /* Check for QSFP interrupt after t_init (SFF 8679)
5886 * + extra
5887 */
5888 msleep(3000);
5889 if (!ppd->qsfp_info.qsfp_interrupt_functional) {
5890 if (do_qsfp_intr_fallback(ppd) < 0)
5891 dd_dev_info(dd,
5892 "%s: QSFP fallback failed\n",
5893 __func__);
5894 ppd->driver_link_ready = 1;
5895 }
5896 }
5897}
5898
bbdeb33d
DL		 5899/*
5900 * Do a one-time initialize of the LCB block.
5901 */
5902static void init_lcb(struct hfi1_devdata *dd)
5903{
5904 /* the DC has been reset earlier in the driver load */
5905
5906 /* set LCB for cclk loopback on the port */
5907 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
5908 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
5909 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
5910 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
5911 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
5912 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
5913 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
5914}
5915
77241056
MM		 5916int bringup_serdes(struct hfi1_pportdata *ppd)
5917{
5918 struct hfi1_devdata *dd = ppd->dd;
5919 u64 guid;
5920 int ret;
5921
5922 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
5923 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
5924
5925 guid = ppd->guid;
5926 if (!guid) {
5927 if (dd->base_guid)
5928 guid = dd->base_guid + ppd->port - 1;
5929 ppd->guid = guid;
5930 }
5931
5932 /* the link defaults to enabled */
5933 ppd->link_enabled = 1;
5934 /* Set linkinit_reason on power up per OPA spec */
5935 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
5936
bbdeb33d
DL		 5937 /* one-time init of the LCB */
5938 init_lcb(dd);
5939
77241056
MM		 5940 if (loopback) {
5941 ret = init_loopback(dd);
5942 if (ret < 0)
5943 return ret;
5944 }
5945
5946 return start_link(ppd);
5947}
5948
5949void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
5950{
5951 struct hfi1_devdata *dd = ppd->dd;
5952
5953 /*
5954 * Shut down the link and keep it down. First turn off that the
5955 * driver wants to allow the link to be up (driver_link_ready).
5956 * Then make sure the link is not automatically restarted
5957 * (link_enabled). Cancel any pending restart. And finally
5958 * go offline.
5959 */
5960 ppd->driver_link_ready = 0;
5961 ppd->link_enabled = 0;
5962
5963 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
5964 OPA_LINKDOWN_REASON_SMA_DISABLED);
5965 set_link_state(ppd, HLS_DN_OFFLINE);
5966
5967 /* disable the port */
5968 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
5969}
5970
5971static inline int init_cpu_counters(struct hfi1_devdata *dd)
5972{
5973 struct hfi1_pportdata *ppd;
5974 int i;
5975
5976 ppd = (struct hfi1_pportdata *)(dd + 1);
5977 for (i = 0; i < dd->num_pports; i++, ppd++) {
5978 ppd->ibport_data.rc_acks = NULL;
5979 ppd->ibport_data.rc_qacks = NULL;
5980 ppd->ibport_data.rc_acks = alloc_percpu(u64);
5981 ppd->ibport_data.rc_qacks = alloc_percpu(u64);
5982 ppd->ibport_data.rc_delayed_comp = alloc_percpu(u64);
5983 if ((ppd->ibport_data.rc_acks == NULL) ||
5984 (ppd->ibport_data.rc_delayed_comp == NULL) ||
5985 (ppd->ibport_data.rc_qacks == NULL))
5986 return -ENOMEM;
5987 }
5988
5989 return 0;
5990}
5991
5992static const char * const pt_names[] = {
5993 "expected",
5994 "eager",
5995 "invalid"
5996};
5997
5998static const char *pt_name(u32 type)
5999{
6000 return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
6001}
6002
6003/*
6004 * index is the index into the receive array
6005 */
6006void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
6007 u32 type, unsigned long pa, u16 order)
6008{
6009 u64 reg;
6010 void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
6011 (dd->kregbase + RCV_ARRAY));
6012
6013 if (!(dd->flags & HFI1_PRESENT))
6014 goto done;
6015
6016 if (type == PT_INVALID) {
6017 pa = 0;
6018 } else if (type > PT_INVALID) {
6019 dd_dev_err(dd,
6020 "unexpected receive array type %u for index %u, not handled\n",
6021 type, index);
6022 goto done;
6023 }
6024
6025 hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
6026 pt_name(type), index, pa, (unsigned long)order);
6027
6028#define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
6029 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
6030 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
6031 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
6032 << RCV_ARRAY_RT_ADDR_SHIFT;
6033 writeq(reg, base + (index * 8));
6034
6035 if (type == PT_EAGER)
6036 /*
6037 * Eager entries are written one-by-one so we have to push them
6038 * after we write the entry.
6039 */
6040 flush_wc();
6041done:
6042 return;
6043}
6044
6045void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
6046{
6047 struct hfi1_devdata *dd = rcd->dd;
6048 u32 i;
6049
6050 /* this could be optimized */
6051 for (i = rcd->eager_base; i < rcd->eager_base +
6052 rcd->egrbufs.alloced; i++)
6053 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
6054
6055 for (i = rcd->expected_base;
6056 i < rcd->expected_base + rcd->expected_count; i++)
6057 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
6058}
6059
6060int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
6061 struct hfi1_ctxt_info *kinfo)
6062{
6063 kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
6064 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
6065 return 0;
6066}
6067
6068struct hfi1_message_header *hfi1_get_msgheader(
6069 struct hfi1_devdata *dd, __le32 *rhf_addr)
6070{
6071 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
6072
6073 return (struct hfi1_message_header *)
6074 (rhf_addr - dd->rhf_offset + offset);
6075}
6076
6077static const char * const ib_cfg_name_strings[] = {
6078 "HFI1_IB_CFG_LIDLMC",
6079 "HFI1_IB_CFG_LWID_DG_ENB",
6080 "HFI1_IB_CFG_LWID_ENB",
6081 "HFI1_IB_CFG_LWID",
6082 "HFI1_IB_CFG_SPD_ENB",
6083 "HFI1_IB_CFG_SPD",
6084 "HFI1_IB_CFG_RXPOL_ENB",
6085 "HFI1_IB_CFG_LREV_ENB",
6086 "HFI1_IB_CFG_LINKLATENCY",
6087 "HFI1_IB_CFG_HRTBT",
6088 "HFI1_IB_CFG_OP_VLS",
6089 "HFI1_IB_CFG_VL_HIGH_CAP",
6090 "HFI1_IB_CFG_VL_LOW_CAP",
6091 "HFI1_IB_CFG_OVERRUN_THRESH",
6092 "HFI1_IB_CFG_PHYERR_THRESH",
6093 "HFI1_IB_CFG_LINKDEFAULT",
6094 "HFI1_IB_CFG_PKEYS",
6095 "HFI1_IB_CFG_MTU",
6096 "HFI1_IB_CFG_LSTATE",
6097 "HFI1_IB_CFG_VL_HIGH_LIMIT",
6098 "HFI1_IB_CFG_PMA_TICKS",
6099 "HFI1_IB_CFG_PORT"
6100};
6101
6102static const char *ib_cfg_name(int which)
6103{
6104 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
6105 return "invalid";
6106 return ib_cfg_name_strings[which];
6107}
6108
6109int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
6110{
6111 struct hfi1_devdata *dd = ppd->dd;
6112 int val = 0;
6113
6114 switch (which) {
6115 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
6116 val = ppd->link_width_enabled;
6117 break;
6118 case HFI1_IB_CFG_LWID: /* currently active Link-width */
6119 val = ppd->link_width_active;
6120 break;
6121 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
6122 val = ppd->link_speed_enabled;
6123 break;
6124 case HFI1_IB_CFG_SPD: /* current Link speed */
6125 val = ppd->link_speed_active;
6126 break;
6127
6128 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
6129 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
6130 case HFI1_IB_CFG_LINKLATENCY:
6131 goto unimplemented;
6132
6133 case HFI1_IB_CFG_OP_VLS:
6134 val = ppd->vls_operational;
6135 break;
6136 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
6137 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
6138 break;
6139 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
6140 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
6141 break;
6142 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
6143 val = ppd->overrun_threshold;
6144 break;
6145 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
6146 val = ppd->phy_error_threshold;
6147 break;
6148 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
6149 val = dd->link_default;
6150 break;
6151
6152 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
6153 case HFI1_IB_CFG_PMA_TICKS:
6154 default:
6155unimplemented:
6156 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
6157 dd_dev_info(
6158 dd,
6159 "%s: which %s: not implemented\n",
6160 __func__,
6161 ib_cfg_name(which));
6162 break;
6163 }
6164
6165 return val;
6166}
6167
6168/*
6169 * The largest MAD packet size.
6170 */
6171#define MAX_MAD_PACKET 2048
6172
6173/*
6174 * Return the maximum header bytes that can go on the _wire_
6175 * for this device. This count includes the ICRC which is
6176 * not part of the packet held in memory but it is appended
6177 * by the HW.
6178 * This is dependent on the device's receive header entry size.
6179 * HFI allows this to be set per-receive context, but the
6180 * driver presently enforces a global value.
6181 */
6182u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
6183{
6184 /*
6185 * The maximum non-payload (MTU) bytes in LRH.PktLen are
6186 * the Receive Header Entry Size minus the PBC (or RHF) size
6187 * plus one DW for the ICRC appended by HW.
6188 *
6189 * dd->rcd[0].rcvhdrqentsize is in DW.
6190 * We use rcd[0] as all context will have the same value. Also,
6191 * the first kernel context would have been allocated by now so
6192 * we are guaranteed a valid value.
6193 */
6194 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
6195}
6196
6197/*
6198 * Set Send Length
6199 * @ppd - per port data
6200 *
6201 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
6202 * registers compare against LRH.PktLen, so use the max bytes included
6203 * in the LRH.
6204 *
6205 * This routine changes all VL values except VL15, which it maintains at
6206 * the same value.
6207 */
6208static void set_send_length(struct hfi1_pportdata *ppd)
6209{
6210 struct hfi1_devdata *dd = ppd->dd;
6211 u32 max_hb = lrh_max_header_bytes(dd), maxvlmtu = 0, dcmtu;
6212 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
6213 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
6214 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
6215 int i;
6216
6217 for (i = 0; i < ppd->vls_supported; i++) {
6218 if (dd->vld[i].mtu > maxvlmtu)
6219 maxvlmtu = dd->vld[i].mtu;
6220 if (i <= 3)
6221 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
6222 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
6223 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
6224 else
6225 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
6226 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
6227 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
6228 }
6229 write_csr(dd, SEND_LEN_CHECK0, len1);
6230 write_csr(dd, SEND_LEN_CHECK1, len2);
6231 /* adjust kernel credit return thresholds based on new MTUs */
6232 /* all kernel receive contexts have the same hdrqentsize */
6233 for (i = 0; i < ppd->vls_supported; i++) {
6234 sc_set_cr_threshold(dd->vld[i].sc,
6235 sc_mtu_to_threshold(dd->vld[i].sc, dd->vld[i].mtu,
6236 dd->rcd[0]->rcvhdrqentsize));
6237 }
6238 sc_set_cr_threshold(dd->vld[15].sc,
6239 sc_mtu_to_threshold(dd->vld[15].sc, dd->vld[15].mtu,
6240 dd->rcd[0]->rcvhdrqentsize));
6241
6242 /* Adjust maximum MTU for the port in DC */
6243 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
6244 (ilog2(maxvlmtu >> 8) + 1);
6245 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
6246 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
6247 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
6248 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
6249 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
6250}
6251
6252static void set_lidlmc(struct hfi1_pportdata *ppd)
6253{
6254 int i;
6255 u64 sreg = 0;
6256 struct hfi1_devdata *dd = ppd->dd;
6257 u32 mask = ~((1U << ppd->lmc) - 1);
6258 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
6259
6260 if (dd->hfi1_snoop.mode_flag)
6261 dd_dev_info(dd, "Set lid/lmc while snooping");
6262
6263 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
6264 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
6265 c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
6266 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT)|
6267 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
6268 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
6269 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
6270
6271 /*
6272 * Iterate over all the send contexts and set their SLID check
6273 */
6274 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
6275 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
6276 (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
6277 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
6278
6279 for (i = 0; i < dd->chip_send_contexts; i++) {
6280 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
6281 i, (u32)sreg);
6282 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
6283 }
6284
6285 /* Now we have to do the same thing for the sdma engines */
6286 sdma_update_lmc(dd, mask, ppd->lid);
6287}
6288
6289static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
6290{
6291 unsigned long timeout;
6292 u32 curr_state;
6293
6294 timeout = jiffies + msecs_to_jiffies(msecs);
6295 while (1) {
6296 curr_state = read_physical_state(dd);
6297 if (curr_state == state)
6298 break;
6299 if (time_after(jiffies, timeout)) {
6300 dd_dev_err(dd,
6301 "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
6302 state, curr_state);
6303 return -ETIMEDOUT;
6304 }
6305 usleep_range(1950, 2050); /* sleep 2ms-ish */
6306 }
6307
6308 return 0;
6309}
6310
6311/*
6312 * Helper for set_link_state(). Do not call except from that routine.
6313 * Expects ppd->hls_mutex to be held.
6314 *
6315 * @rem_reason value to be sent to the neighbor
6316 *
6317 * LinkDownReasons only set if transition succeeds.
6318 */
6319static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
6320{
6321 struct hfi1_devdata *dd = ppd->dd;
6322 u32 pstate, previous_state;
6323 u32 last_local_state;
6324 u32 last_remote_state;
6325 int ret;
6326 int do_transition;
6327 int do_wait;
6328
6329 previous_state = ppd->host_link_state;
6330 ppd->host_link_state = HLS_GOING_OFFLINE;
6331 pstate = read_physical_state(dd);
6332 if (pstate == PLS_OFFLINE) {
6333 do_transition = 0; /* in right state */
6334 do_wait = 0; /* ...no need to wait */
6335 } else if ((pstate & 0xff) == PLS_OFFLINE) {
6336 do_transition = 0; /* in an offline transient state */
6337 do_wait = 1; /* ...wait for it to settle */
6338 } else {
6339 do_transition = 1; /* need to move to offline */
6340 do_wait = 1; /* ...will need to wait */
6341 }
6342
6343 if (do_transition) {
6344 ret = set_physical_link_state(dd,
6345 PLS_OFFLINE | (rem_reason << 8));
6346
6347 if (ret != HCMD_SUCCESS) {
6348 dd_dev_err(dd,
6349 "Failed to transition to Offline link state, return %d\n",
6350 ret);
6351 return -EINVAL;
6352 }
6353 if (ppd->offline_disabled_reason == OPA_LINKDOWN_REASON_NONE)
6354 ppd->offline_disabled_reason =
6355 OPA_LINKDOWN_REASON_TRANSIENT;
6356 }
6357
6358 if (do_wait) {
6359 /* it can take a while for the link to go down */
dc060245 6360 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
77241056
MM		 6361 if (ret < 0)
6362 return ret;
6363 }
6364
6365 /* make sure the logical state is also down */
6366 wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
6367
6368 /*
6369 * Now in charge of LCB - must be after the physical state is
6370 * offline.quiet and before host_link_state is changed.
6371 */
6372 set_host_lcb_access(dd);
6373 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
6374 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
6375
6376 /*
6377 * The LNI has a mandatory wait time after the physical state
6378 * moves to Offline.Quiet. The wait time may be different
6379 * depending on how the link went down. The 8051 firmware
6380 * will observe the needed wait time and only move to ready
6381 * when that is completed. The largest of the quiet timeouts
05087f3b
DL		 6382 * is 6s, so wait that long and then at least 0.5s more for
6383 * other transitions, and another 0.5s for a buffer.
77241056 6384 */
05087f3b 6385 ret = wait_fm_ready(dd, 7000);
77241056
MM		 6386 if (ret) {
6387 dd_dev_err(dd,
6388 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
6389 /* state is really offline, so make it so */
6390 ppd->host_link_state = HLS_DN_OFFLINE;
6391 return ret;
6392 }
6393
6394 /*
6395 * The state is now offline and the 8051 is ready to accept host
6396 * requests.
6397 * - change our state
6398 * - notify others if we were previously in a linkup state
6399 */
6400 ppd->host_link_state = HLS_DN_OFFLINE;
6401 if (previous_state & HLS_UP) {
6402 /* went down while link was up */
6403 handle_linkup_change(dd, 0);
6404 } else if (previous_state
6405 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
6406 /* went down while attempting link up */
6407 /* byte 1 of last_*_state is the failure reason */
6408 read_last_local_state(dd, &last_local_state);
6409 read_last_remote_state(dd, &last_remote_state);
6410 dd_dev_err(dd,
6411 "LNI failure last states: local 0x%08x, remote 0x%08x\n",
6412 last_local_state, last_remote_state);
6413 }
6414
6415 /* the active link width (downgrade) is 0 on link down */
6416 ppd->link_width_active = 0;
6417 ppd->link_width_downgrade_tx_active = 0;
6418 ppd->link_width_downgrade_rx_active = 0;
6419 ppd->current_egress_rate = 0;
6420 return 0;
6421}
6422
6423/* return the link state name */
6424static const char *link_state_name(u32 state)
6425{
6426 const char *name;
6427 int n = ilog2(state);
6428 static const char * const names[] = {
6429 [__HLS_UP_INIT_BP] = "INIT",
6430 [__HLS_UP_ARMED_BP] = "ARMED",
6431 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
6432 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
6433 [__HLS_DN_POLL_BP] = "POLL",
6434 [__HLS_DN_DISABLE_BP] = "DISABLE",
6435 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
6436 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
6437 [__HLS_GOING_UP_BP] = "GOING_UP",
6438 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
6439 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
6440 };
6441
6442 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
6443 return name ? name : "unknown";
6444}
6445
6446/* return the link state reason name */
6447static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
6448{
6449 if (state == HLS_UP_INIT) {
6450 switch (ppd->linkinit_reason) {
6451 case OPA_LINKINIT_REASON_LINKUP:
6452 return "(LINKUP)";
6453 case OPA_LINKINIT_REASON_FLAPPING:
6454 return "(FLAPPING)";
6455 case OPA_LINKINIT_OUTSIDE_POLICY:
6456 return "(OUTSIDE_POLICY)";
6457 case OPA_LINKINIT_QUARANTINED:
6458 return "(QUARANTINED)";
6459 case OPA_LINKINIT_INSUFIC_CAPABILITY:
6460 return "(INSUFIC_CAPABILITY)";
6461 default:
6462 break;
6463 }
6464 }
6465 return "";
6466}
6467
6468/*
6469 * driver_physical_state - convert the driver's notion of a port's
6470 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
6471 * Return -1 (converted to a u32) to indicate error.
6472 */
6473u32 driver_physical_state(struct hfi1_pportdata *ppd)
6474{
6475 switch (ppd->host_link_state) {
6476 case HLS_UP_INIT:
6477 case HLS_UP_ARMED:
6478 case HLS_UP_ACTIVE:
6479 return IB_PORTPHYSSTATE_LINKUP;
6480 case HLS_DN_POLL:
6481 return IB_PORTPHYSSTATE_POLLING;
6482 case HLS_DN_DISABLE:
6483 return IB_PORTPHYSSTATE_DISABLED;
6484 case HLS_DN_OFFLINE:
6485 return OPA_PORTPHYSSTATE_OFFLINE;
6486 case HLS_VERIFY_CAP:
6487 return IB_PORTPHYSSTATE_POLLING;
6488 case HLS_GOING_UP:
6489 return IB_PORTPHYSSTATE_POLLING;
6490 case HLS_GOING_OFFLINE:
6491 return OPA_PORTPHYSSTATE_OFFLINE;
6492 case HLS_LINK_COOLDOWN:
6493 return OPA_PORTPHYSSTATE_OFFLINE;
6494 case HLS_DN_DOWNDEF:
6495 default:
6496 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
6497 ppd->host_link_state);
6498 return -1;
6499 }
6500}
6501
6502/*
6503 * driver_logical_state - convert the driver's notion of a port's
6504 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
6505 * (converted to a u32) to indicate error.
6506 */
6507u32 driver_logical_state(struct hfi1_pportdata *ppd)
6508{
6509 if (ppd->host_link_state && !(ppd->host_link_state & HLS_UP))
6510 return IB_PORT_DOWN;
6511
6512 switch (ppd->host_link_state & HLS_UP) {
6513 case HLS_UP_INIT:
6514 return IB_PORT_INIT;
6515 case HLS_UP_ARMED:
6516 return IB_PORT_ARMED;
6517 case HLS_UP_ACTIVE:
6518 return IB_PORT_ACTIVE;
6519 default:
6520 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
6521 ppd->host_link_state);
6522 return -1;
6523 }
6524}
6525
6526void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
6527 u8 neigh_reason, u8 rem_reason)
6528{
6529 if (ppd->local_link_down_reason.latest == 0 &&
6530 ppd->neigh_link_down_reason.latest == 0) {
6531 ppd->local_link_down_reason.latest = lcl_reason;
6532 ppd->neigh_link_down_reason.latest = neigh_reason;
6533 ppd->remote_link_down_reason = rem_reason;
6534 }
6535}
6536
6537/*
6538 * Change the physical and/or logical link state.
6539 *
6540 * Do not call this routine while inside an interrupt. It contains
6541 * calls to routines that can take multiple seconds to finish.
6542 *
6543 * Returns 0 on success, -errno on failure.
6544 */
6545int set_link_state(struct hfi1_pportdata *ppd, u32 state)
6546{
6547 struct hfi1_devdata *dd = ppd->dd;
6548 struct ib_event event = {.device = NULL};
6549 int ret1, ret = 0;
6550 int was_up, is_down;
6551 int orig_new_state, poll_bounce;
6552
6553 mutex_lock(&ppd->hls_lock);
6554
6555 orig_new_state = state;
6556 if (state == HLS_DN_DOWNDEF)
6557 state = dd->link_default;
6558
6559 /* interpret poll -> poll as a link bounce */
6560 poll_bounce = ppd->host_link_state == HLS_DN_POLL
6561 && state == HLS_DN_POLL;
6562
6563 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
6564 link_state_name(ppd->host_link_state),
6565 link_state_name(orig_new_state),
6566 poll_bounce ? "(bounce) " : "",
6567 link_state_reason_name(ppd, state));
6568
6569 was_up = !!(ppd->host_link_state & HLS_UP);
6570
6571 /*
6572 * If we're going to a (HLS_*) link state that implies the logical
6573 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
6574 * reset is_sm_config_started to 0.
6575 */
6576 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
6577 ppd->is_sm_config_started = 0;
6578
6579 /*
6580 * Do nothing if the states match. Let a poll to poll link bounce
6581 * go through.
6582 */
6583 if (ppd->host_link_state == state && !poll_bounce)
6584 goto done;
6585
6586 switch (state) {
6587 case HLS_UP_INIT:
6588 if (ppd->host_link_state == HLS_DN_POLL && (quick_linkup
6589 || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
6590 /*
6591 * Quick link up jumps from polling to here.
6592 *
6593 * Whether in normal or loopback mode, the
6594 * simulator jumps from polling to link up.
6595 * Accept that here.
6596 */
6597 /* OK */;
6598 } else if (ppd->host_link_state != HLS_GOING_UP) {
6599 goto unexpected;
6600 }
6601
6602 ppd->host_link_state = HLS_UP_INIT;
6603 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
6604 if (ret) {
6605 /* logical state didn't change, stay at going_up */
6606 ppd->host_link_state = HLS_GOING_UP;
6607 dd_dev_err(dd,
6608 "%s: logical state did not change to INIT\n",
6609 __func__);
6610 } else {
6611 /* clear old transient LINKINIT_REASON code */
6612 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
6613 ppd->linkinit_reason =
6614 OPA_LINKINIT_REASON_LINKUP;
6615
6616 /* enable the port */
6617 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6618
6619 handle_linkup_change(dd, 1);
6620 }
6621 break;
6622 case HLS_UP_ARMED:
6623 if (ppd->host_link_state != HLS_UP_INIT)
6624 goto unexpected;
6625
6626 ppd->host_link_state = HLS_UP_ARMED;
6627 set_logical_state(dd, LSTATE_ARMED);
6628 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
6629 if (ret) {
6630 /* logical state didn't change, stay at init */
6631 ppd->host_link_state = HLS_UP_INIT;
6632 dd_dev_err(dd,
6633 "%s: logical state did not change to ARMED\n",
6634 __func__);
6635 }
6636 /*
6637 * The simulator does not currently implement SMA messages,
6638 * so neighbor_normal is not set. Set it here when we first
6639 * move to Armed.
6640 */
6641 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
6642 ppd->neighbor_normal = 1;
6643 break;
6644 case HLS_UP_ACTIVE:
6645 if (ppd->host_link_state != HLS_UP_ARMED)
6646 goto unexpected;
6647
6648 ppd->host_link_state = HLS_UP_ACTIVE;
6649 set_logical_state(dd, LSTATE_ACTIVE);
6650 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
6651 if (ret) {
6652 /* logical state didn't change, stay at armed */
6653 ppd->host_link_state = HLS_UP_ARMED;
6654 dd_dev_err(dd,
6655 "%s: logical state did not change to ACTIVE\n",
6656 __func__);
6657 } else {
6658
6659 /* tell all engines to go running */
6660 sdma_all_running(dd);
6661
6662 /* Signal the IB layer that the port has went active */
6663 event.device = &dd->verbs_dev.ibdev;
6664 event.element.port_num = ppd->port;
6665 event.event = IB_EVENT_PORT_ACTIVE;
6666 }
6667 break;
6668 case HLS_DN_POLL:
6669 if ((ppd->host_link_state == HLS_DN_DISABLE ||
6670 ppd->host_link_state == HLS_DN_OFFLINE) &&
6671 dd->dc_shutdown)
6672 dc_start(dd);
6673 /* Hand LED control to the DC */
6674 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
6675
6676 if (ppd->host_link_state != HLS_DN_OFFLINE) {
6677 u8 tmp = ppd->link_enabled;
6678
6679 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6680 if (ret) {
6681 ppd->link_enabled = tmp;
6682 break;
6683 }
6684 ppd->remote_link_down_reason = 0;
6685
6686 if (ppd->driver_link_ready)
6687 ppd->link_enabled = 1;
6688 }
6689
6690 ret = set_local_link_attributes(ppd);
6691 if (ret)
6692 break;
6693
6694 ppd->port_error_action = 0;
6695 ppd->host_link_state = HLS_DN_POLL;
6696
6697 if (quick_linkup) {
6698 /* quick linkup does not go into polling */
6699 ret = do_quick_linkup(dd);
6700 } else {
6701 ret1 = set_physical_link_state(dd, PLS_POLLING);
6702 if (ret1 != HCMD_SUCCESS) {
6703 dd_dev_err(dd,
6704 "Failed to transition to Polling link state, return 0x%x\n",
6705 ret1);
6706 ret = -EINVAL;
6707 }
6708 }
6709 ppd->offline_disabled_reason = OPA_LINKDOWN_REASON_NONE;
6710 /*
6711 * If an error occurred above, go back to offline. The
6712 * caller may reschedule another attempt.
6713 */
6714 if (ret)
6715 goto_offline(ppd, 0);
6716 break;
6717 case HLS_DN_DISABLE:
6718 /* link is disabled */
6719 ppd->link_enabled = 0;
6720
6721 /* allow any state to transition to disabled */
6722
6723 /* must transition to offline first */
6724 if (ppd->host_link_state != HLS_DN_OFFLINE) {
6725 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6726 if (ret)
6727 break;
6728 ppd->remote_link_down_reason = 0;
6729 }
6730
6731 ret1 = set_physical_link_state(dd, PLS_DISABLED);
6732 if (ret1 != HCMD_SUCCESS) {
6733 dd_dev_err(dd,
6734 "Failed to transition to Disabled link state, return 0x%x\n",
6735 ret1);
6736 ret = -EINVAL;
6737 break;
6738 }
6739 ppd->host_link_state = HLS_DN_DISABLE;
6740 dc_shutdown(dd);
6741 break;
6742 case HLS_DN_OFFLINE:
6743 if (ppd->host_link_state == HLS_DN_DISABLE)
6744 dc_start(dd);
6745
6746 /* allow any state to transition to offline */
6747 ret = goto_offline(ppd, ppd->remote_link_down_reason);
6748 if (!ret)
6749 ppd->remote_link_down_reason = 0;
6750 break;
6751 case HLS_VERIFY_CAP:
6752 if (ppd->host_link_state != HLS_DN_POLL)
6753 goto unexpected;
6754 ppd->host_link_state = HLS_VERIFY_CAP;
6755 break;
6756 case HLS_GOING_UP:
6757 if (ppd->host_link_state != HLS_VERIFY_CAP)
6758 goto unexpected;
6759
6760 ret1 = set_physical_link_state(dd, PLS_LINKUP);
6761 if (ret1 != HCMD_SUCCESS) {
6762 dd_dev_err(dd,
6763 "Failed to transition to link up state, return 0x%x\n",
6764 ret1);
6765 ret = -EINVAL;
6766 break;
6767 }
6768 ppd->host_link_state = HLS_GOING_UP;
6769 break;
6770
6771 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
6772 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
6773 default:
6774 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
6775 __func__, state);
6776 ret = -EINVAL;
6777 break;
6778 }
6779
6780 is_down = !!(ppd->host_link_state & (HLS_DN_POLL |
6781 HLS_DN_DISABLE | HLS_DN_OFFLINE));
6782
6783 if (was_up && is_down && ppd->local_link_down_reason.sma == 0 &&
6784 ppd->neigh_link_down_reason.sma == 0) {
6785 ppd->local_link_down_reason.sma =
6786 ppd->local_link_down_reason.latest;
6787 ppd->neigh_link_down_reason.sma =
6788 ppd->neigh_link_down_reason.latest;
6789 }
6790
6791 goto done;
6792
6793unexpected:
6794 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
6795 __func__, link_state_name(ppd->host_link_state),
6796 link_state_name(state));
6797 ret = -EINVAL;
6798
6799done:
6800 mutex_unlock(&ppd->hls_lock);
6801
6802 if (event.device)
6803 ib_dispatch_event(&event);
6804
6805 return ret;
6806}
6807
6808int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
6809{
6810 u64 reg;
6811 int ret = 0;
6812
6813 switch (which) {
6814 case HFI1_IB_CFG_LIDLMC:
6815 set_lidlmc(ppd);
6816 break;
6817 case HFI1_IB_CFG_VL_HIGH_LIMIT:
6818 /*
6819 * The VL Arbitrator high limit is sent in units of 4k
6820 * bytes, while HFI stores it in units of 64 bytes.
6821 */
6822 val *= 4096/64;
6823 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
6824 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
6825 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
6826 break;
6827 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
6828 /* HFI only supports POLL as the default link down state */
6829 if (val != HLS_DN_POLL)
6830 ret = -EINVAL;
6831 break;
6832 case HFI1_IB_CFG_OP_VLS:
6833 if (ppd->vls_operational != val) {
6834 ppd->vls_operational = val;
6835 if (!ppd->port)
6836 ret = -EINVAL;
6837 else
6838 ret = sdma_map_init(
6839 ppd->dd,
6840 ppd->port - 1,
6841 val,
6842 NULL);
6843 }
6844 break;
6845 /*
6846 * For link width, link width downgrade, and speed enable, always AND
6847 * the setting with what is actually supported. This has two benefits.
6848 * First, enabled can't have unsupported values, no matter what the
6849 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
6850 * "fill in with your supported value" have all the bits in the
6851 * field set, so simply ANDing with supported has the desired result.
6852 */
6853 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
6854 ppd->link_width_enabled = val & ppd->link_width_supported;
6855 break;
6856 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
6857 ppd->link_width_downgrade_enabled =
6858 val & ppd->link_width_downgrade_supported;
6859 break;
6860 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
6861 ppd->link_speed_enabled = val & ppd->link_speed_supported;
6862 break;
6863 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
6864 /*
6865 * HFI does not follow IB specs, save this value
6866 * so we can report it, if asked.
6867 */
6868 ppd->overrun_threshold = val;
6869 break;
6870 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
6871 /*
6872 * HFI does not follow IB specs, save this value
6873 * so we can report it, if asked.
6874 */
6875 ppd->phy_error_threshold = val;
6876 break;
6877
6878 case HFI1_IB_CFG_MTU:
6879 set_send_length(ppd);
6880 break;
6881
6882 case HFI1_IB_CFG_PKEYS:
6883 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
6884 set_partition_keys(ppd);
6885 break;
6886
6887 default:
6888 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
6889 dd_dev_info(ppd->dd,
6890 "%s: which %s, val 0x%x: not implemented\n",
6891 __func__, ib_cfg_name(which), val);
6892 break;
6893 }
6894 return ret;
6895}
6896
6897/* begin functions related to vl arbitration table caching */
6898static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
6899{
6900 int i;
6901
6902 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
6903 VL_ARB_LOW_PRIO_TABLE_SIZE);
6904 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
6905 VL_ARB_HIGH_PRIO_TABLE_SIZE);
6906
6907 /*
6908 * Note that we always return values directly from the
6909 * 'vl_arb_cache' (and do no CSR reads) in response to a
6910 * 'Get(VLArbTable)'. This is obviously correct after a
6911 * 'Set(VLArbTable)', since the cache will then be up to
6912 * date. But it's also correct prior to any 'Set(VLArbTable)'
6913 * since then both the cache, and the relevant h/w registers
6914 * will be zeroed.
6915 */
6916
6917 for (i = 0; i < MAX_PRIO_TABLE; i++)
6918 spin_lock_init(&ppd->vl_arb_cache[i].lock);
6919}
6920
6921/*
6922 * vl_arb_lock_cache
6923 *
6924 * All other vl_arb_* functions should be called only after locking
6925 * the cache.
6926 */
6927static inline struct vl_arb_cache *
6928vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
6929{
6930 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
6931 return NULL;
6932 spin_lock(&ppd->vl_arb_cache[idx].lock);
6933 return &ppd->vl_arb_cache[idx];
6934}
6935
6936static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
6937{
6938 spin_unlock(&ppd->vl_arb_cache[idx].lock);
6939}
6940
6941static void vl_arb_get_cache(struct vl_arb_cache *cache,
6942 struct ib_vl_weight_elem *vl)
6943{
6944 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
6945}
6946
6947static void vl_arb_set_cache(struct vl_arb_cache *cache,
6948 struct ib_vl_weight_elem *vl)
6949{
6950 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
6951}
6952
6953static int vl_arb_match_cache(struct vl_arb_cache *cache,
6954 struct ib_vl_weight_elem *vl)
6955{
6956 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
6957}
6958/* end functions related to vl arbitration table caching */
6959
6960static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
6961 u32 size, struct ib_vl_weight_elem *vl)
6962{
6963 struct hfi1_devdata *dd = ppd->dd;
6964 u64 reg;
6965 unsigned int i, is_up = 0;
6966 int drain, ret = 0;
6967
6968 mutex_lock(&ppd->hls_lock);
6969
6970 if (ppd->host_link_state & HLS_UP)
6971 is_up = 1;
6972
6973 drain = !is_ax(dd) && is_up;
6974
6975 if (drain)
6976 /*
6977 * Before adjusting VL arbitration weights, empty per-VL
6978 * FIFOs, otherwise a packet whose VL weight is being
6979 * set to 0 could get stuck in a FIFO with no chance to
6980 * egress.
6981 */
6982 ret = stop_drain_data_vls(dd);
6983
6984 if (ret) {
6985 dd_dev_err(
6986 dd,
6987 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
6988 __func__);
6989 goto err;
6990 }
6991
6992 for (i = 0; i < size; i++, vl++) {
6993 /*
6994 * NOTE: The low priority shift and mask are used here, but
6995 * they are the same for both the low and high registers.
6996 */
6997 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
6998 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
6999 | (((u64)vl->weight
7000 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
7001 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
7002 write_csr(dd, target + (i * 8), reg);
7003 }
7004 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
7005
7006 if (drain)
7007 open_fill_data_vls(dd); /* reopen all VLs */
7008
7009err:
7010 mutex_unlock(&ppd->hls_lock);
7011
7012 return ret;
7013}
7014
7015/*
7016 * Read one credit merge VL register.
7017 */
7018static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
7019 struct vl_limit *vll)
7020{
7021 u64 reg = read_csr(dd, csr);
7022
7023 vll->dedicated = cpu_to_be16(
7024 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
7025 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
7026 vll->shared = cpu_to_be16(
7027 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
7028 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
7029}
7030
7031/*
7032 * Read the current credit merge limits.
7033 */
7034static int get_buffer_control(struct hfi1_devdata *dd,
7035 struct buffer_control *bc, u16 *overall_limit)
7036{
7037 u64 reg;
7038 int i;
7039
7040 /* not all entries are filled in */
7041 memset(bc, 0, sizeof(*bc));
7042
7043 /* OPA and HFI have a 1-1 mapping */
7044 for (i = 0; i < TXE_NUM_DATA_VL; i++)
7045 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8*i), &bc->vl[i]);
7046
7047 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
7048 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
7049
7050 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7051 bc->overall_shared_limit = cpu_to_be16(
7052 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
7053 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
7054 if (overall_limit)
7055 *overall_limit = (reg
7056 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
7057 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
7058 return sizeof(struct buffer_control);
7059}
7060
7061static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
7062{
7063 u64 reg;
7064 int i;
7065
7066 /* each register contains 16 SC->VLnt mappings, 4 bits each */
7067 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
7068 for (i = 0; i < sizeof(u64); i++) {
7069 u8 byte = *(((u8 *)&reg) + i);
7070
7071 dp->vlnt[2 * i] = byte & 0xf;
7072 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
7073 }
7074
7075 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
7076 for (i = 0; i < sizeof(u64); i++) {
7077 u8 byte = *(((u8 *)&reg) + i);
7078
7079 dp->vlnt[16 + (2 * i)] = byte & 0xf;
7080 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
7081 }
7082 return sizeof(struct sc2vlnt);
7083}
7084
7085static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
7086 struct ib_vl_weight_elem *vl)
7087{
7088 unsigned int i;
7089
7090 for (i = 0; i < nelems; i++, vl++) {
7091 vl->vl = 0xf;
7092 vl->weight = 0;
7093 }
7094}
7095
7096static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
7097{
7098 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
7099 DC_SC_VL_VAL(15_0,
7100 0, dp->vlnt[0] & 0xf,
7101 1, dp->vlnt[1] & 0xf,
7102 2, dp->vlnt[2] & 0xf,
7103 3, dp->vlnt[3] & 0xf,
7104 4, dp->vlnt[4] & 0xf,
7105 5, dp->vlnt[5] & 0xf,
7106 6, dp->vlnt[6] & 0xf,
7107 7, dp->vlnt[7] & 0xf,
7108 8, dp->vlnt[8] & 0xf,
7109 9, dp->vlnt[9] & 0xf,
7110 10, dp->vlnt[10] & 0xf,
7111 11, dp->vlnt[11] & 0xf,
7112 12, dp->vlnt[12] & 0xf,
7113 13, dp->vlnt[13] & 0xf,
7114 14, dp->vlnt[14] & 0xf,
7115 15, dp->vlnt[15] & 0xf));
7116 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
7117 DC_SC_VL_VAL(31_16,
7118 16, dp->vlnt[16] & 0xf,
7119 17, dp->vlnt[17] & 0xf,
7120 18, dp->vlnt[18] & 0xf,
7121 19, dp->vlnt[19] & 0xf,
7122 20, dp->vlnt[20] & 0xf,
7123 21, dp->vlnt[21] & 0xf,
7124 22, dp->vlnt[22] & 0xf,
7125 23, dp->vlnt[23] & 0xf,
7126 24, dp->vlnt[24] & 0xf,
7127 25, dp->vlnt[25] & 0xf,
7128 26, dp->vlnt[26] & 0xf,
7129 27, dp->vlnt[27] & 0xf,
7130 28, dp->vlnt[28] & 0xf,
7131 29, dp->vlnt[29] & 0xf,
7132 30, dp->vlnt[30] & 0xf,
7133 31, dp->vlnt[31] & 0xf));
7134}
7135
7136static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
7137 u16 limit)
7138{
7139 if (limit != 0)
7140 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
7141 what, (int)limit, idx);
7142}
7143
7144/* change only the shared limit portion of SendCmGLobalCredit */
7145static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
7146{
7147 u64 reg;
7148
7149 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7150 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
7151 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
7152 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
7153}
7154
7155/* change only the total credit limit portion of SendCmGLobalCredit */
7156static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
7157{
7158 u64 reg;
7159
7160 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
7161 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
7162 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
7163 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
7164}
7165
7166/* set the given per-VL shared limit */
7167static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
7168{
7169 u64 reg;
7170 u32 addr;
7171
7172 if (vl < TXE_NUM_DATA_VL)
7173 addr = SEND_CM_CREDIT_VL + (8 * vl);
7174 else
7175 addr = SEND_CM_CREDIT_VL15;
7176
7177 reg = read_csr(dd, addr);
7178 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
7179 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
7180 write_csr(dd, addr, reg);
7181}
7182
7183/* set the given per-VL dedicated limit */
7184static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
7185{
7186 u64 reg;
7187 u32 addr;
7188
7189 if (vl < TXE_NUM_DATA_VL)
7190 addr = SEND_CM_CREDIT_VL + (8 * vl);
7191 else
7192 addr = SEND_CM_CREDIT_VL15;
7193
7194 reg = read_csr(dd, addr);
7195 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
7196 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
7197 write_csr(dd, addr, reg);
7198}
7199
7200/* spin until the given per-VL status mask bits clear */
7201static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
7202 const char *which)
7203{
7204 unsigned long timeout;
7205 u64 reg;
7206
7207 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
7208 while (1) {
7209 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
7210
7211 if (reg == 0)
7212 return; /* success */
7213 if (time_after(jiffies, timeout))
7214 break; /* timed out */
7215 udelay(1);
7216 }
7217
7218 dd_dev_err(dd,
7219 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
7220 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
7221 /*
7222 * If this occurs, it is likely there was a credit loss on the link.
7223 * The only recovery from that is a link bounce.
7224 */
7225 dd_dev_err(dd,
7226 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
7227}
7228
7229/*
7230 * The number of credits on the VLs may be changed while everything
7231 * is "live", but the following algorithm must be followed due to
7232 * how the hardware is actually implemented. In particular,
7233 * Return_Credit_Status[] is the only correct status check.
7234 *
7235 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
7236 * set Global_Shared_Credit_Limit = 0
7237 * use_all_vl = 1
7238 * mask0 = all VLs that are changing either dedicated or shared limits
7239 * set Shared_Limit[mask0] = 0
7240 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
7241 * if (changing any dedicated limit)
7242 * mask1 = all VLs that are lowering dedicated limits
7243 * lower Dedicated_Limit[mask1]
7244 * spin until Return_Credit_Status[mask1] == 0
7245 * raise Dedicated_Limits
7246 * raise Shared_Limits
7247 * raise Global_Shared_Credit_Limit
7248 *
7249 * lower = if the new limit is lower, set the limit to the new value
7250 * raise = if the new limit is higher than the current value (may be changed
7251 * earlier in the algorithm), set the new limit to the new value
7252 */
7253static int set_buffer_control(struct hfi1_devdata *dd,
7254 struct buffer_control *new_bc)
7255{
7256 u64 changing_mask, ld_mask, stat_mask;
7257 int change_count;
7258 int i, use_all_mask;
7259 int this_shared_changing;
7260 /*
7261 * A0: add the variable any_shared_limit_changing below and in the
7262 * algorithm above. If removing A0 support, it can be removed.
7263 */
7264 int any_shared_limit_changing;
7265 struct buffer_control cur_bc;
7266 u8 changing[OPA_MAX_VLS];
7267 u8 lowering_dedicated[OPA_MAX_VLS];
7268 u16 cur_total;
7269 u32 new_total = 0;
7270 const u64 all_mask =
7271 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
7272 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
7273 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
7274 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
7275 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
7276 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
7277 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
7278 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
7279 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
7280
7281#define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
7282#define NUM_USABLE_VLS 16 /* look at VL15 and less */
7283
7284
7285 /* find the new total credits, do sanity check on unused VLs */
7286 for (i = 0; i < OPA_MAX_VLS; i++) {
7287 if (valid_vl(i)) {
7288 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
7289 continue;
7290 }
7291 nonzero_msg(dd, i, "dedicated",
7292 be16_to_cpu(new_bc->vl[i].dedicated));
7293 nonzero_msg(dd, i, "shared",
7294 be16_to_cpu(new_bc->vl[i].shared));
7295 new_bc->vl[i].dedicated = 0;
7296 new_bc->vl[i].shared = 0;
7297 }
7298 new_total += be16_to_cpu(new_bc->overall_shared_limit);
7299 if (new_total > (u32)dd->link_credits)
7300 return -EINVAL;
7301 /* fetch the current values */
7302 get_buffer_control(dd, &cur_bc, &cur_total);
7303
7304 /*
7305 * Create the masks we will use.
7306 */
7307 memset(changing, 0, sizeof(changing));
7308 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
7309 /* NOTE: Assumes that the individual VL bits are adjacent and in
7310 increasing order */
7311 stat_mask =
7312 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
7313 changing_mask = 0;
7314 ld_mask = 0;
7315 change_count = 0;
7316 any_shared_limit_changing = 0;
7317 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
7318 if (!valid_vl(i))
7319 continue;
7320 this_shared_changing = new_bc->vl[i].shared
7321 != cur_bc.vl[i].shared;
7322 if (this_shared_changing)
7323 any_shared_limit_changing = 1;
7324 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated
7325 || this_shared_changing) {
7326 changing[i] = 1;
7327 changing_mask |= stat_mask;
7328 change_count++;
7329 }
7330 if (be16_to_cpu(new_bc->vl[i].dedicated) <
7331 be16_to_cpu(cur_bc.vl[i].dedicated)) {
7332 lowering_dedicated[i] = 1;
7333 ld_mask |= stat_mask;
7334 }
7335 }
7336
7337 /* bracket the credit change with a total adjustment */
7338 if (new_total > cur_total)
7339 set_global_limit(dd, new_total);
7340
7341 /*
7342 * Start the credit change algorithm.
7343 */
7344 use_all_mask = 0;
7345 if ((be16_to_cpu(new_bc->overall_shared_limit) <
995deafa
MM		 7346 be16_to_cpu(cur_bc.overall_shared_limit)) ||
7347 (is_ax(dd) && any_shared_limit_changing)) {
77241056
MM		 7348 set_global_shared(dd, 0);
7349 cur_bc.overall_shared_limit = 0;
7350 use_all_mask = 1;
7351 }
7352
7353 for (i = 0; i < NUM_USABLE_VLS; i++) {
7354 if (!valid_vl(i))
7355 continue;
7356
7357 if (changing[i]) {
7358 set_vl_shared(dd, i, 0);
7359 cur_bc.vl[i].shared = 0;
7360 }
7361 }
7362
7363 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
7364 "shared");
7365
7366 if (change_count > 0) {
7367 for (i = 0; i < NUM_USABLE_VLS; i++) {
7368 if (!valid_vl(i))
7369 continue;
7370
7371 if (lowering_dedicated[i]) {
7372 set_vl_dedicated(dd, i,
7373 be16_to_cpu(new_bc->vl[i].dedicated));
7374 cur_bc.vl[i].dedicated =
7375 new_bc->vl[i].dedicated;
7376 }
7377 }
7378
7379 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
7380
7381 /* now raise all dedicated that are going up */
7382 for (i = 0; i < NUM_USABLE_VLS; i++) {
7383 if (!valid_vl(i))
7384 continue;
7385
7386 if (be16_to_cpu(new_bc->vl[i].dedicated) >
7387 be16_to_cpu(cur_bc.vl[i].dedicated))
7388 set_vl_dedicated(dd, i,
7389 be16_to_cpu(new_bc->vl[i].dedicated));
7390 }
7391 }
7392
7393 /* next raise all shared that are going up */
7394 for (i = 0; i < NUM_USABLE_VLS; i++) {
7395 if (!valid_vl(i))
7396 continue;
7397
7398 if (be16_to_cpu(new_bc->vl[i].shared) >
7399 be16_to_cpu(cur_bc.vl[i].shared))
7400 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
7401 }
7402
7403 /* finally raise the global shared */
7404 if (be16_to_cpu(new_bc->overall_shared_limit) >
7405 be16_to_cpu(cur_bc.overall_shared_limit))
7406 set_global_shared(dd,
7407 be16_to_cpu(new_bc->overall_shared_limit));
7408
7409 /* bracket the credit change with a total adjustment */
7410 if (new_total < cur_total)
7411 set_global_limit(dd, new_total);
7412 return 0;
7413}
7414
7415/*
7416 * Read the given fabric manager table. Return the size of the
7417 * table (in bytes) on success, and a negative error code on
7418 * failure.
7419 */
7420int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
7421
7422{
7423 int size;
7424 struct vl_arb_cache *vlc;
7425
7426 switch (which) {
7427 case FM_TBL_VL_HIGH_ARB:
7428 size = 256;
7429 /*
7430 * OPA specifies 128 elements (of 2 bytes each), though
7431 * HFI supports only 16 elements in h/w.
7432 */
7433 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
7434 vl_arb_get_cache(vlc, t);
7435 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7436 break;
7437 case FM_TBL_VL_LOW_ARB:
7438 size = 256;
7439 /*
7440 * OPA specifies 128 elements (of 2 bytes each), though
7441 * HFI supports only 16 elements in h/w.
7442 */
7443 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
7444 vl_arb_get_cache(vlc, t);
7445 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7446 break;
7447 case FM_TBL_BUFFER_CONTROL:
7448 size = get_buffer_control(ppd->dd, t, NULL);
7449 break;
7450 case FM_TBL_SC2VLNT:
7451 size = get_sc2vlnt(ppd->dd, t);
7452 break;
7453 case FM_TBL_VL_PREEMPT_ELEMS:
7454 size = 256;
7455 /* OPA specifies 128 elements, of 2 bytes each */
7456 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
7457 break;
7458 case FM_TBL_VL_PREEMPT_MATRIX:
7459 size = 256;
7460 /*
7461 * OPA specifies that this is the same size as the VL
7462 * arbitration tables (i.e., 256 bytes).
7463 */
7464 break;
7465 default:
7466 return -EINVAL;
7467 }
7468 return size;
7469}
7470
7471/*
7472 * Write the given fabric manager table.
7473 */
7474int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
7475{
7476 int ret = 0;
7477 struct vl_arb_cache *vlc;
7478
7479 switch (which) {
7480 case FM_TBL_VL_HIGH_ARB:
7481 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
7482 if (vl_arb_match_cache(vlc, t)) {
7483 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7484 break;
7485 }
7486 vl_arb_set_cache(vlc, t);
7487 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
7488 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
7489 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
7490 break;
7491 case FM_TBL_VL_LOW_ARB:
7492 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
7493 if (vl_arb_match_cache(vlc, t)) {
7494 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7495 break;
7496 }
7497 vl_arb_set_cache(vlc, t);
7498 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
7499 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
7500 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
7501 break;
7502 case FM_TBL_BUFFER_CONTROL:
7503 ret = set_buffer_control(ppd->dd, t);
7504 break;
7505 case FM_TBL_SC2VLNT:
7506 set_sc2vlnt(ppd->dd, t);
7507 break;
7508 default:
7509 ret = -EINVAL;
7510 }
7511 return ret;
7512}
7513
7514/*
7515 * Disable all data VLs.
7516 *
7517 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
7518 */
7519static int disable_data_vls(struct hfi1_devdata *dd)
7520{
995deafa 7521 if (is_ax(dd))
77241056
MM		 7522 return 1;
7523
7524 pio_send_control(dd, PSC_DATA_VL_DISABLE);
7525
7526 return 0;
7527}
7528
7529/*
7530 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
7531 * Just re-enables all data VLs (the "fill" part happens
7532 * automatically - the name was chosen for symmetry with
7533 * stop_drain_data_vls()).
7534 *
7535 * Return 0 if successful, non-zero if the VLs cannot be enabled.
7536 */
7537int open_fill_data_vls(struct hfi1_devdata *dd)
7538{
995deafa 7539 if (is_ax(dd))
77241056
MM		 7540 return 1;
7541
7542 pio_send_control(dd, PSC_DATA_VL_ENABLE);
7543
7544 return 0;
7545}
7546
7547/*
7548 * drain_data_vls() - assumes that disable_data_vls() has been called,
7549 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
7550 * engines to drop to 0.
7551 */
7552static void drain_data_vls(struct hfi1_devdata *dd)
7553{
7554 sc_wait(dd);
7555 sdma_wait(dd);
7556 pause_for_credit_return(dd);
7557}
7558
7559/*
7560 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
7561 *
7562 * Use open_fill_data_vls() to resume using data VLs. This pair is
7563 * meant to be used like this:
7564 *
7565 * stop_drain_data_vls(dd);
7566 * // do things with per-VL resources
7567 * open_fill_data_vls(dd);
7568 */
7569int stop_drain_data_vls(struct hfi1_devdata *dd)
7570{
7571 int ret;
7572
7573 ret = disable_data_vls(dd);
7574 if (ret == 0)
7575 drain_data_vls(dd);
7576
7577 return ret;
7578}
7579
7580/*
7581 * Convert a nanosecond time to a cclock count. No matter how slow
7582 * the cclock, a non-zero ns will always have a non-zero result.
7583 */
7584u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
7585{
7586 u32 cclocks;
7587
7588 if (dd->icode == ICODE_FPGA_EMULATION)
7589 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
7590 else /* simulation pretends to be ASIC */
7591 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
7592 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
7593 cclocks = 1;
7594 return cclocks;
7595}
7596
7597/*
7598 * Convert a cclock count to nanoseconds. Not matter how slow
7599 * the cclock, a non-zero cclocks will always have a non-zero result.
7600 */
7601u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
7602{
7603 u32 ns;
7604
7605 if (dd->icode == ICODE_FPGA_EMULATION)
7606 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
7607 else /* simulation pretends to be ASIC */
7608 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
7609 if (cclocks && !ns)
7610 ns = 1;
7611 return ns;
7612}
7613
7614/*
7615 * Dynamically adjust the receive interrupt timeout for a context based on
7616 * incoming packet rate.
7617 *
7618 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
7619 */
7620static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
7621{
7622 struct hfi1_devdata *dd = rcd->dd;
7623 u32 timeout = rcd->rcvavail_timeout;
7624
7625 /*
7626 * This algorithm doubles or halves the timeout depending on whether
7627 * the number of packets received in this interrupt were less than or
7628 * greater equal the interrupt count.
7629 *
7630 * The calculations below do not allow a steady state to be achieved.
7631 * Only at the endpoints it is possible to have an unchanging
7632 * timeout.
7633 */
7634 if (npkts < rcv_intr_count) {
7635 /*
7636 * Not enough packets arrived before the timeout, adjust
7637 * timeout downward.
7638 */
7639 if (timeout < 2) /* already at minimum? */
7640 return;
7641 timeout >>= 1;
7642 } else {
7643 /*
7644 * More than enough packets arrived before the timeout, adjust
7645 * timeout upward.
7646 */
7647 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
7648 return;
7649 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
7650 }
7651
7652 rcd->rcvavail_timeout = timeout;
7653 /* timeout cannot be larger than rcv_intr_timeout_csr which has already
7654 been verified to be in range */
7655 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
7656 (u64)timeout << RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
7657}
7658
7659void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
7660 u32 intr_adjust, u32 npkts)
7661{
7662 struct hfi1_devdata *dd = rcd->dd;
7663 u64 reg;
7664 u32 ctxt = rcd->ctxt;
7665
7666 /*
7667 * Need to write timeout register before updating RcvHdrHead to ensure
7668 * that a new value is used when the HW decides to restart counting.
7669 */
7670 if (intr_adjust)
7671 adjust_rcv_timeout(rcd, npkts);
7672 if (updegr) {
7673 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
7674 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
7675 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
7676 }
7677 mmiowb();
7678 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
7679 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
7680 << RCV_HDR_HEAD_HEAD_SHIFT);
7681 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
7682 mmiowb();
7683}
7684
7685u32 hdrqempty(struct hfi1_ctxtdata *rcd)
7686{
7687 u32 head, tail;
7688
7689 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
7690 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
7691
7692 if (rcd->rcvhdrtail_kvaddr)
7693 tail = get_rcvhdrtail(rcd);
7694 else
7695 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
7696
7697 return head == tail;
7698}
7699
7700/*
7701 * Context Control and Receive Array encoding for buffer size:
7702 * 0x0 invalid
7703 * 0x1 4 KB
7704 * 0x2 8 KB
7705 * 0x3 16 KB
7706 * 0x4 32 KB
7707 * 0x5 64 KB
7708 * 0x6 128 KB
7709 * 0x7 256 KB
7710 * 0x8 512 KB (Receive Array only)
7711 * 0x9 1 MB (Receive Array only)
7712 * 0xa 2 MB (Receive Array only)
7713 *
7714 * 0xB-0xF - reserved (Receive Array only)
7715 *
7716 *
7717 * This routine assumes that the value has already been sanity checked.
7718 */
7719static u32 encoded_size(u32 size)
7720{
7721 switch (size) {
7722 case 4*1024: return 0x1;
7723 case 8*1024: return 0x2;
7724 case 16*1024: return 0x3;
7725 case 32*1024: return 0x4;
7726 case 64*1024: return 0x5;
7727 case 128*1024: return 0x6;
7728 case 256*1024: return 0x7;
7729 case 512*1024: return 0x8;
7730 case 1*1024*1024: return 0x9;
7731 case 2*1024*1024: return 0xa;
7732 }
7733 return 0x1; /* if invalid, go with the minimum size */
7734}
7735
7736void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
7737{
7738 struct hfi1_ctxtdata *rcd;
7739 u64 rcvctrl, reg;
7740 int did_enable = 0;
7741
7742 rcd = dd->rcd[ctxt];
7743 if (!rcd)
7744 return;
7745
7746 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
7747
7748 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
7749 /* if the context already enabled, don't do the extra steps */
7750 if ((op & HFI1_RCVCTRL_CTXT_ENB)
7751 && !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
7752 /* reset the tail and hdr addresses, and sequence count */
7753 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
7754 rcd->rcvhdrq_phys);
7755 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
7756 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7757 rcd->rcvhdrqtailaddr_phys);
7758 rcd->seq_cnt = 1;
7759
7760 /* reset the cached receive header queue head value */
7761 rcd->head = 0;
7762
7763 /*
7764 * Zero the receive header queue so we don't get false
7765 * positives when checking the sequence number. The
7766 * sequence numbers could land exactly on the same spot.
7767 * E.g. a rcd restart before the receive header wrapped.
7768 */
7769 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
7770
7771 /* starting timeout */
7772 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
7773
7774 /* enable the context */
7775 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
7776
7777 /* clean the egr buffer size first */
7778 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
7779 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
7780 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
7781 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
7782
7783 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
7784 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
7785 did_enable = 1;
7786
7787 /* zero RcvEgrIndexHead */
7788 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
7789
7790 /* set eager count and base index */
7791 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
7792 & RCV_EGR_CTRL_EGR_CNT_MASK)
7793 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
7794 (((rcd->eager_base >> RCV_SHIFT)
7795 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
7796 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
7797 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
7798
7799 /*
7800 * Set TID (expected) count and base index.
7801 * rcd->expected_count is set to individual RcvArray entries,
7802 * not pairs, and the CSR takes a pair-count in groups of
7803 * four, so divide by 8.
7804 */
7805 reg = (((rcd->expected_count >> RCV_SHIFT)
7806 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
7807 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
7808 (((rcd->expected_base >> RCV_SHIFT)
7809 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
7810 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
7811 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
82c2611d
NV		 7812 if (ctxt == HFI1_CTRL_CTXT)
7813 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
77241056
MM		 7814 }
7815 if (op & HFI1_RCVCTRL_CTXT_DIS) {
7816 write_csr(dd, RCV_VL15, 0);
46b010d3
MB		 7817 /*
7818 * When receive context is being disabled turn on tail
7819 * update with a dummy tail address and then disable
7820 * receive context.
7821 */
7822 if (dd->rcvhdrtail_dummy_physaddr) {
7823 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7824 dd->rcvhdrtail_dummy_physaddr);
7825 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7826 }
7827
77241056
MM		 7828 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
7829 }
7830 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
7831 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
7832 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
7833 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
7834 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
7835 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7836 if (op & HFI1_RCVCTRL_TAILUPD_DIS)
7837 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
7838 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
7839 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
7840 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
7841 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
7842 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
7843 /* In one-packet-per-eager mode, the size comes from
7844 the RcvArray entry. */
7845 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
7846 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
7847 }
7848 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
7849 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
7850 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
7851 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
7852 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
7853 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
7854 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
7855 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
7856 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
7857 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
7858 rcd->rcvctrl = rcvctrl;
7859 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
7860 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
7861
7862 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
7863 if (did_enable
7864 && (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
7865 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
7866 if (reg != 0) {
7867 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
7868 ctxt, reg);
7869 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
7870 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
7871 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
7872 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
7873 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
7874 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
7875 ctxt, reg, reg == 0 ? "not" : "still");
7876 }
7877 }
7878
7879 if (did_enable) {
7880 /*
7881 * The interrupt timeout and count must be set after
7882 * the context is enabled to take effect.
7883 */
7884 /* set interrupt timeout */
7885 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
7886 (u64)rcd->rcvavail_timeout <<
7887 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
7888
7889 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
7890 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
7891 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
7892 }
7893
7894 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
7895 /*
7896 * If the context has been disabled and the Tail Update has
46b010d3
MB		 7897 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
7898 * so it doesn't contain an address that is invalid.
77241056 7899 */
46b010d3
MB		 7900 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
7901 dd->rcvhdrtail_dummy_physaddr);
77241056
MM		 7902}
7903
7904u32 hfi1_read_cntrs(struct hfi1_devdata *dd, loff_t pos, char **namep,
7905 u64 **cntrp)
7906{
7907 int ret;
7908 u64 val = 0;
7909
7910 if (namep) {
7911 ret = dd->cntrnameslen;
7912 if (pos != 0) {
7913 dd_dev_err(dd, "read_cntrs does not support indexing");
7914 return 0;
7915 }
7916 *namep = dd->cntrnames;
7917 } else {
7918 const struct cntr_entry *entry;
7919 int i, j;
7920
7921 ret = (dd->ndevcntrs) * sizeof(u64);
7922 if (pos != 0) {
7923 dd_dev_err(dd, "read_cntrs does not support indexing");
7924 return 0;
7925 }
7926
7927 /* Get the start of the block of counters */
7928 *cntrp = dd->cntrs;
7929
7930 /*
7931 * Now go and fill in each counter in the block.
7932 */
7933 for (i = 0; i < DEV_CNTR_LAST; i++) {
7934 entry = &dev_cntrs[i];
7935 hfi1_cdbg(CNTR, "reading %s", entry->name);
7936 if (entry->flags & CNTR_DISABLED) {
7937 /* Nothing */
7938 hfi1_cdbg(CNTR, "\tDisabled\n");
7939 } else {
7940 if (entry->flags & CNTR_VL) {
7941 hfi1_cdbg(CNTR, "\tPer VL\n");
7942 for (j = 0; j < C_VL_COUNT; j++) {
7943 val = entry->rw_cntr(entry,
7944 dd, j,
7945 CNTR_MODE_R,
7946 0);
7947 hfi1_cdbg(
7948 CNTR,
7949 "\t\tRead 0x%llx for %d\n",
7950 val, j);
7951 dd->cntrs[entry->offset + j] =
7952 val;
7953 }
7954 } else {
7955 val = entry->rw_cntr(entry, dd,
7956 CNTR_INVALID_VL,
7957 CNTR_MODE_R, 0);
7958 dd->cntrs[entry->offset] = val;
7959 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
7960 }
7961 }
7962 }
7963 }
7964 return ret;
7965}
7966
7967/*
7968 * Used by sysfs to create files for hfi stats to read
7969 */
7970u32 hfi1_read_portcntrs(struct hfi1_devdata *dd, loff_t pos, u32 port,
7971 char **namep, u64 **cntrp)
7972{
7973 int ret;
7974 u64 val = 0;
7975
7976 if (namep) {
7977 ret = dd->portcntrnameslen;
7978 if (pos != 0) {
7979 dd_dev_err(dd, "index not supported");
7980 return 0;
7981 }
7982 *namep = dd->portcntrnames;
7983 } else {
7984 const struct cntr_entry *entry;
7985 struct hfi1_pportdata *ppd;
7986 int i, j;
7987
7988 ret = (dd->nportcntrs) * sizeof(u64);
7989 if (pos != 0) {
7990 dd_dev_err(dd, "indexing not supported");
7991 return 0;
7992 }
7993 ppd = (struct hfi1_pportdata *)(dd + 1 + port);
7994 *cntrp = ppd->cntrs;
7995
7996 for (i = 0; i < PORT_CNTR_LAST; i++) {
7997 entry = &port_cntrs[i];
7998 hfi1_cdbg(CNTR, "reading %s", entry->name);
7999 if (entry->flags & CNTR_DISABLED) {
8000 /* Nothing */
8001 hfi1_cdbg(CNTR, "\tDisabled\n");
8002 continue;
8003 }
8004
8005 if (entry->flags & CNTR_VL) {
8006 hfi1_cdbg(CNTR, "\tPer VL");
8007 for (j = 0; j < C_VL_COUNT; j++) {
8008 val = entry->rw_cntr(entry, ppd, j,
8009 CNTR_MODE_R,
8010 0);
8011 hfi1_cdbg(
8012 CNTR,
8013 "\t\tRead 0x%llx for %d",
8014 val, j);
8015 ppd->cntrs[entry->offset + j] = val;
8016 }
8017 } else {
8018 val = entry->rw_cntr(entry, ppd,
8019 CNTR_INVALID_VL,
8020 CNTR_MODE_R,
8021 0);
8022 ppd->cntrs[entry->offset] = val;
8023 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
8024 }
8025 }
8026 }
8027 return ret;
8028}
8029
8030static void free_cntrs(struct hfi1_devdata *dd)
8031{
8032 struct hfi1_pportdata *ppd;
8033 int i;
8034
8035 if (dd->synth_stats_timer.data)
8036 del_timer_sync(&dd->synth_stats_timer);
8037 dd->synth_stats_timer.data = 0;
8038 ppd = (struct hfi1_pportdata *)(dd + 1);
8039 for (i = 0; i < dd->num_pports; i++, ppd++) {
8040 kfree(ppd->cntrs);
8041 kfree(ppd->scntrs);
8042 free_percpu(ppd->ibport_data.rc_acks);
8043 free_percpu(ppd->ibport_data.rc_qacks);
8044 free_percpu(ppd->ibport_data.rc_delayed_comp);
8045 ppd->cntrs = NULL;
8046 ppd->scntrs = NULL;
8047 ppd->ibport_data.rc_acks = NULL;
8048 ppd->ibport_data.rc_qacks = NULL;
8049 ppd->ibport_data.rc_delayed_comp = NULL;
8050 }
8051 kfree(dd->portcntrnames);
8052 dd->portcntrnames = NULL;
8053 kfree(dd->cntrs);
8054 dd->cntrs = NULL;
8055 kfree(dd->scntrs);
8056 dd->scntrs = NULL;
8057 kfree(dd->cntrnames);
8058 dd->cntrnames = NULL;
8059}
8060
8061#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
8062#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
8063
8064static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
8065 u64 *psval, void *context, int vl)
8066{
8067 u64 val;
8068 u64 sval = *psval;
8069
8070 if (entry->flags & CNTR_DISABLED) {
8071 dd_dev_err(dd, "Counter %s not enabled", entry->name);
8072 return 0;
8073 }
8074
8075 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
8076
8077 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
8078
8079 /* If its a synthetic counter there is more work we need to do */
8080 if (entry->flags & CNTR_SYNTH) {
8081 if (sval == CNTR_MAX) {
8082 /* No need to read already saturated */
8083 return CNTR_MAX;
8084 }
8085
8086 if (entry->flags & CNTR_32BIT) {
8087 /* 32bit counters can wrap multiple times */
8088 u64 upper = sval >> 32;
8089 u64 lower = (sval << 32) >> 32;
8090
8091 if (lower > val) { /* hw wrapped */
8092 if (upper == CNTR_32BIT_MAX)
8093 val = CNTR_MAX;
8094 else
8095 upper++;
8096 }
8097
8098 if (val != CNTR_MAX)
8099 val = (upper << 32) | val;
8100
8101 } else {
8102 /* If we rolled we are saturated */
8103 if ((val < sval) || (val > CNTR_MAX))
8104 val = CNTR_MAX;
8105 }
8106 }
8107
8108 *psval = val;
8109
8110 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
8111
8112 return val;
8113}
8114
8115static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
8116 struct cntr_entry *entry,
8117 u64 *psval, void *context, int vl, u64 data)
8118{
8119 u64 val;
8120
8121 if (entry->flags & CNTR_DISABLED) {
8122 dd_dev_err(dd, "Counter %s not enabled", entry->name);
8123 return 0;
8124 }
8125
8126 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
8127
8128 if (entry->flags & CNTR_SYNTH) {
8129 *psval = data;
8130 if (entry->flags & CNTR_32BIT) {
8131 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
8132 (data << 32) >> 32);
8133 val = data; /* return the full 64bit value */
8134 } else {
8135 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
8136 data);
8137 }
8138 } else {
8139 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
8140 }
8141
8142 *psval = val;
8143
8144 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
8145
8146 return val;
8147}
8148
8149u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
8150{
8151 struct cntr_entry *entry;
8152 u64 *sval;
8153
8154 entry = &dev_cntrs[index];
8155 sval = dd->scntrs + entry->offset;
8156
8157 if (vl != CNTR_INVALID_VL)
8158 sval += vl;
8159
8160 return read_dev_port_cntr(dd, entry, sval, dd, vl);
8161}
8162
8163u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
8164{
8165 struct cntr_entry *entry;
8166 u64 *sval;
8167
8168 entry = &dev_cntrs[index];
8169 sval = dd->scntrs + entry->offset;
8170
8171 if (vl != CNTR_INVALID_VL)
8172 sval += vl;
8173
8174 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
8175}
8176
8177u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
8178{
8179 struct cntr_entry *entry;
8180 u64 *sval;
8181
8182 entry = &port_cntrs[index];
8183 sval = ppd->scntrs + entry->offset;
8184
8185 if (vl != CNTR_INVALID_VL)
8186 sval += vl;
8187
8188 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
8189 (index <= C_RCV_HDR_OVF_LAST)) {
8190 /* We do not want to bother for disabled contexts */
8191 return 0;
8192 }
8193
8194 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
8195}
8196
8197u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
8198{
8199 struct cntr_entry *entry;
8200 u64 *sval;
8201
8202 entry = &port_cntrs[index];
8203 sval = ppd->scntrs + entry->offset;
8204
8205 if (vl != CNTR_INVALID_VL)
8206 sval += vl;
8207
8208 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
8209 (index <= C_RCV_HDR_OVF_LAST)) {
8210 /* We do not want to bother for disabled contexts */
8211 return 0;
8212 }
8213
8214 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
8215}
8216
8217static void update_synth_timer(unsigned long opaque)
8218{
8219 u64 cur_tx;
8220 u64 cur_rx;
8221 u64 total_flits;
8222 u8 update = 0;
8223 int i, j, vl;
8224 struct hfi1_pportdata *ppd;
8225 struct cntr_entry *entry;
8226
8227 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
8228
8229 /*
8230 * Rather than keep beating on the CSRs pick a minimal set that we can
8231 * check to watch for potential roll over. We can do this by looking at
8232 * the number of flits sent/recv. If the total flits exceeds 32bits then
8233 * we have to iterate all the counters and update.
8234 */
8235 entry = &dev_cntrs[C_DC_RCV_FLITS];
8236 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
8237
8238 entry = &dev_cntrs[C_DC_XMIT_FLITS];
8239 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
8240
8241 hfi1_cdbg(
8242 CNTR,
8243 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
8244 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
8245
8246 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
8247 /*
8248 * May not be strictly necessary to update but it won't hurt and
8249 * simplifies the logic here.
8250 */
8251 update = 1;
8252 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
8253 dd->unit);
8254 } else {
8255 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
8256 hfi1_cdbg(CNTR,
8257 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
8258 total_flits, (u64)CNTR_32BIT_MAX);
8259 if (total_flits >= CNTR_32BIT_MAX) {
8260 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
8261 dd->unit);
8262 update = 1;
8263 }
8264 }
8265
8266 if (update) {
8267 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
8268 for (i = 0; i < DEV_CNTR_LAST; i++) {
8269 entry = &dev_cntrs[i];
8270 if (entry->flags & CNTR_VL) {
8271 for (vl = 0; vl < C_VL_COUNT; vl++)
8272 read_dev_cntr(dd, i, vl);
8273 } else {
8274 read_dev_cntr(dd, i, CNTR_INVALID_VL);
8275 }
8276 }
8277 ppd = (struct hfi1_pportdata *)(dd + 1);
8278 for (i = 0; i < dd->num_pports; i++, ppd++) {
8279 for (j = 0; j < PORT_CNTR_LAST; j++) {
8280 entry = &port_cntrs[j];
8281 if (entry->flags & CNTR_VL) {
8282 for (vl = 0; vl < C_VL_COUNT; vl++)
8283 read_port_cntr(ppd, j, vl);
8284 } else {
8285 read_port_cntr(ppd, j, CNTR_INVALID_VL);
8286 }
8287 }
8288 }
8289
8290 /*
8291 * We want the value in the register. The goal is to keep track
8292 * of the number of "ticks" not the counter value. In other
8293 * words if the register rolls we want to notice it and go ahead
8294 * and force an update.
8295 */
8296 entry = &dev_cntrs[C_DC_XMIT_FLITS];
8297 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
8298 CNTR_MODE_R, 0);
8299
8300 entry = &dev_cntrs[C_DC_RCV_FLITS];
8301 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
8302 CNTR_MODE_R, 0);
8303
8304 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
8305 dd->unit, dd->last_tx, dd->last_rx);
8306
8307 } else {
8308 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
8309 }
8310
8311mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
8312}
8313
8314#define C_MAX_NAME 13 /* 12 chars + one for /0 */
8315static int init_cntrs(struct hfi1_devdata *dd)
8316{
8317 int i, rcv_ctxts, index, j;
8318 size_t sz;
8319 char *p;
8320 char name[C_MAX_NAME];
8321 struct hfi1_pportdata *ppd;
8322
8323 /* set up the stats timer; the add_timer is done at the end */
24523a94
MFW		 8324 setup_timer(&dd->synth_stats_timer, update_synth_timer,
8325 (unsigned long)dd);
77241056
MM		 8326
8327 /***********************/
8328 /* per device counters */
8329 /***********************/
8330
8331 /* size names and determine how many we have*/
8332 dd->ndevcntrs = 0;
8333 sz = 0;
8334 index = 0;
8335
8336 for (i = 0; i < DEV_CNTR_LAST; i++) {
8337 hfi1_dbg_early("Init cntr %s\n", dev_cntrs[i].name);
8338 if (dev_cntrs[i].flags & CNTR_DISABLED) {
8339 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
8340 continue;
8341 }
8342
8343 if (dev_cntrs[i].flags & CNTR_VL) {
8344 hfi1_dbg_early("\tProcessing VL cntr\n");
8345 dev_cntrs[i].offset = index;
8346 for (j = 0; j < C_VL_COUNT; j++) {
8347 memset(name, '\0', C_MAX_NAME);
8348 snprintf(name, C_MAX_NAME, "%s%d",
8349 dev_cntrs[i].name,
8350 vl_from_idx(j));
8351 sz += strlen(name);
8352 sz++;
8353 hfi1_dbg_early("\t\t%s\n", name);
8354 dd->ndevcntrs++;
8355 index++;
8356 }
8357 } else {
8358 /* +1 for newline */
8359 sz += strlen(dev_cntrs[i].name) + 1;
8360 dd->ndevcntrs++;
8361 dev_cntrs[i].offset = index;
8362 index++;
8363 hfi1_dbg_early("\tAdding %s\n", dev_cntrs[i].name);
8364 }
8365 }
8366
8367 /* allocate space for the counter values */
8368 dd->cntrs = kcalloc(index, sizeof(u64), GFP_KERNEL);
8369 if (!dd->cntrs)
8370 goto bail;
8371
8372 dd->scntrs = kcalloc(index, sizeof(u64), GFP_KERNEL);
8373 if (!dd->scntrs)
8374 goto bail;
8375
8376
8377 /* allocate space for the counter names */
8378 dd->cntrnameslen = sz;
8379 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
8380 if (!dd->cntrnames)
8381 goto bail;
8382
8383 /* fill in the names */
8384 for (p = dd->cntrnames, i = 0, index = 0; i < DEV_CNTR_LAST; i++) {
8385 if (dev_cntrs[i].flags & CNTR_DISABLED) {
8386 /* Nothing */
8387 } else {
8388 if (dev_cntrs[i].flags & CNTR_VL) {
8389 for (j = 0; j < C_VL_COUNT; j++) {
8390 memset(name, '\0', C_MAX_NAME);
8391 snprintf(name, C_MAX_NAME, "%s%d",
8392 dev_cntrs[i].name,
8393 vl_from_idx(j));
8394 memcpy(p, name, strlen(name));
8395 p += strlen(name);
8396 *p++ = '\n';
8397 }
8398 } else {
8399 memcpy(p, dev_cntrs[i].name,
8400 strlen(dev_cntrs[i].name));
8401 p += strlen(dev_cntrs[i].name);
8402 *p++ = '\n';
8403 }
8404 index++;
8405 }
8406 }
8407
8408 /*********************/
8409 /* per port counters */
8410 /*********************/
8411
8412 /*
8413 * Go through the counters for the overflows and disable the ones we
8414 * don't need. This varies based on platform so we need to do it
8415 * dynamically here.
8416 */
8417 rcv_ctxts = dd->num_rcv_contexts;
8418 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
8419 i <= C_RCV_HDR_OVF_LAST; i++) {
8420 port_cntrs[i].flags |= CNTR_DISABLED;
8421 }
8422
8423 /* size port counter names and determine how many we have*/
8424 sz = 0;
8425 dd->nportcntrs = 0;
8426 for (i = 0; i < PORT_CNTR_LAST; i++) {
8427 hfi1_dbg_early("Init pcntr %s\n", port_cntrs[i].name);
8428 if (port_cntrs[i].flags & CNTR_DISABLED) {
8429 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
8430 continue;
8431 }
8432
8433 if (port_cntrs[i].flags & CNTR_VL) {
8434 hfi1_dbg_early("\tProcessing VL cntr\n");
8435 port_cntrs[i].offset = dd->nportcntrs;
8436 for (j = 0; j < C_VL_COUNT; j++) {
8437 memset(name, '\0', C_MAX_NAME);
8438 snprintf(name, C_MAX_NAME, "%s%d",
8439 port_cntrs[i].name,
8440 vl_from_idx(j));
8441 sz += strlen(name);
8442 sz++;
8443 hfi1_dbg_early("\t\t%s\n", name);
8444 dd->nportcntrs++;
8445 }
8446 } else {
8447 /* +1 for newline */
8448 sz += strlen(port_cntrs[i].name) + 1;
8449 port_cntrs[i].offset = dd->nportcntrs;
8450 dd->nportcntrs++;
8451 hfi1_dbg_early("\tAdding %s\n", port_cntrs[i].name);
8452 }
8453 }
8454
8455 /* allocate space for the counter names */
8456 dd->portcntrnameslen = sz;
8457 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
8458 if (!dd->portcntrnames)
8459 goto bail;
8460
8461 /* fill in port cntr names */
8462 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
8463 if (port_cntrs[i].flags & CNTR_DISABLED)
8464 continue;
8465
8466 if (port_cntrs[i].flags & CNTR_VL) {
8467 for (j = 0; j < C_VL_COUNT; j++) {
8468 memset(name, '\0', C_MAX_NAME);
8469 snprintf(name, C_MAX_NAME, "%s%d",
8470 port_cntrs[i].name,
8471 vl_from_idx(j));
8472 memcpy(p, name, strlen(name));
8473 p += strlen(name);
8474 *p++ = '\n';
8475 }
8476 } else {
8477 memcpy(p, port_cntrs[i].name,
8478 strlen(port_cntrs[i].name));
8479 p += strlen(port_cntrs[i].name);
8480 *p++ = '\n';
8481 }
8482 }
8483
8484 /* allocate per port storage for counter values */
8485 ppd = (struct hfi1_pportdata *)(dd + 1);
8486 for (i = 0; i < dd->num_pports; i++, ppd++) {
8487 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
8488 if (!ppd->cntrs)
8489 goto bail;
8490
8491 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
8492 if (!ppd->scntrs)
8493 goto bail;
8494 }
8495
8496 /* CPU counters need to be allocated and zeroed */
8497 if (init_cpu_counters(dd))
8498 goto bail;
8499
8500 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
8501 return 0;
8502bail:
8503 free_cntrs(dd);
8504 return -ENOMEM;
8505}
8506
8507
8508static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
8509{
8510 switch (chip_lstate) {
8511 default:
8512 dd_dev_err(dd,
8513 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
8514 chip_lstate);
8515 /* fall through */
8516 case LSTATE_DOWN:
8517 return IB_PORT_DOWN;
8518 case LSTATE_INIT:
8519 return IB_PORT_INIT;
8520 case LSTATE_ARMED:
8521 return IB_PORT_ARMED;
8522 case LSTATE_ACTIVE:
8523 return IB_PORT_ACTIVE;
8524 }
8525}
8526
8527u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
8528{
8529 /* look at the HFI meta-states only */
8530 switch (chip_pstate & 0xf0) {
8531 default:
8532 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
8533 chip_pstate);
8534 /* fall through */
8535 case PLS_DISABLED:
8536 return IB_PORTPHYSSTATE_DISABLED;
8537 case PLS_OFFLINE:
8538 return OPA_PORTPHYSSTATE_OFFLINE;
8539 case PLS_POLLING:
8540 return IB_PORTPHYSSTATE_POLLING;
8541 case PLS_CONFIGPHY:
8542 return IB_PORTPHYSSTATE_TRAINING;
8543 case PLS_LINKUP:
8544 return IB_PORTPHYSSTATE_LINKUP;
8545 case PLS_PHYTEST:
8546 return IB_PORTPHYSSTATE_PHY_TEST;
8547 }
8548}
8549
8550/* return the OPA port logical state name */
8551const char *opa_lstate_name(u32 lstate)
8552{
8553 static const char * const port_logical_names[] = {
8554 "PORT_NOP",
8555 "PORT_DOWN",
8556 "PORT_INIT",
8557 "PORT_ARMED",
8558 "PORT_ACTIVE",
8559 "PORT_ACTIVE_DEFER",
8560 };
8561 if (lstate < ARRAY_SIZE(port_logical_names))
8562 return port_logical_names[lstate];
8563 return "unknown";
8564}
8565
8566/* return the OPA port physical state name */
8567const char *opa_pstate_name(u32 pstate)
8568{
8569 static const char * const port_physical_names[] = {
8570 "PHYS_NOP",
8571 "reserved1",
8572 "PHYS_POLL",
8573 "PHYS_DISABLED",
8574 "PHYS_TRAINING",
8575 "PHYS_LINKUP",
8576 "PHYS_LINK_ERR_RECOVER",
8577 "PHYS_PHY_TEST",
8578 "reserved8",
8579 "PHYS_OFFLINE",
8580 "PHYS_GANGED",
8581 "PHYS_TEST",
8582 };
8583 if (pstate < ARRAY_SIZE(port_physical_names))
8584 return port_physical_names[pstate];
8585 return "unknown";
8586}
8587
8588/*
8589 * Read the hardware link state and set the driver's cached value of it.
8590 * Return the (new) current value.
8591 */
8592u32 get_logical_state(struct hfi1_pportdata *ppd)
8593{
8594 u32 new_state;
8595
8596 new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
8597 if (new_state != ppd->lstate) {
8598 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
8599 opa_lstate_name(new_state), new_state);
8600 ppd->lstate = new_state;
8601 }
8602 /*
8603 * Set port status flags in the page mapped into userspace
8604 * memory. Do it here to ensure a reliable state - this is
8605 * the only function called by all state handling code.
8606 * Always set the flags due to the fact that the cache value
8607 * might have been changed explicitly outside of this
8608 * function.
8609 */
8610 if (ppd->statusp) {
8611 switch (ppd->lstate) {
8612 case IB_PORT_DOWN:
8613 case IB_PORT_INIT:
8614 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
8615 HFI1_STATUS_IB_READY);
8616 break;
8617 case IB_PORT_ARMED:
8618 *ppd->statusp |= HFI1_STATUS_IB_CONF;
8619 break;
8620 case IB_PORT_ACTIVE:
8621 *ppd->statusp |= HFI1_STATUS_IB_READY;
8622 break;
8623 }
8624 }
8625 return ppd->lstate;
8626}
8627
8628/**
8629 * wait_logical_linkstate - wait for an IB link state change to occur
8630 * @ppd: port device
8631 * @state: the state to wait for
8632 * @msecs: the number of milliseconds to wait
8633 *
8634 * Wait up to msecs milliseconds for IB link state change to occur.
8635 * For now, take the easy polling route.
8636 * Returns 0 if state reached, otherwise -ETIMEDOUT.
8637 */
8638static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
8639 int msecs)
8640{
8641 unsigned long timeout;
8642
8643 timeout = jiffies + msecs_to_jiffies(msecs);
8644 while (1) {
8645 if (get_logical_state(ppd) == state)
8646 return 0;
8647 if (time_after(jiffies, timeout))
8648 break;
8649 msleep(20);
8650 }
8651 dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
8652
8653 return -ETIMEDOUT;
8654}
8655
8656u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
8657{
8658 static u32 remembered_state = 0xff;
8659 u32 pstate;
8660 u32 ib_pstate;
8661
8662 pstate = read_physical_state(ppd->dd);
8663 ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
8664 if (remembered_state != ib_pstate) {
8665 dd_dev_info(ppd->dd,
8666 "%s: physical state changed to %s (0x%x), phy 0x%x\n",
8667 __func__, opa_pstate_name(ib_pstate), ib_pstate,
8668 pstate);
8669 remembered_state = ib_pstate;
8670 }
8671 return ib_pstate;
8672}
8673
8674/*
8675 * Read/modify/write ASIC_QSFP register bits as selected by mask
8676 * data: 0 or 1 in the positions depending on what needs to be written
8677 * dir: 0 for read, 1 for write
8678 * mask: select by setting
8679 * I2CCLK (bit 0)
8680 * I2CDATA (bit 1)
8681 */
8682u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
8683 u32 mask)
8684{
8685 u64 qsfp_oe, target_oe;
8686
8687 target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
8688 if (mask) {
8689 /* We are writing register bits, so lock access */
8690 dir &= mask;
8691 data &= mask;
8692
8693 qsfp_oe = read_csr(dd, target_oe);
8694 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
8695 write_csr(dd, target_oe, qsfp_oe);
8696 }
8697 /* We are exclusively reading bits here, but it is unlikely
8698 * we'll get valid data when we set the direction of the pin
8699 * in the same call, so read should call this function again
8700 * to get valid data
8701 */
8702 return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
8703}
8704
8705#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
8706(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
8707
8708#define SET_STATIC_RATE_CONTROL_SMASK(r) \
8709(r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
8710
8711int hfi1_init_ctxt(struct send_context *sc)
8712{
8713 if (sc != NULL) {
8714 struct hfi1_devdata *dd = sc->dd;
8715 u64 reg;
8716 u8 set = (sc->type == SC_USER ?
8717 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
8718 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
8719 reg = read_kctxt_csr(dd, sc->hw_context,
8720 SEND_CTXT_CHECK_ENABLE);
8721 if (set)
8722 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
8723 else
8724 SET_STATIC_RATE_CONTROL_SMASK(reg);
8725 write_kctxt_csr(dd, sc->hw_context,
8726 SEND_CTXT_CHECK_ENABLE, reg);
8727 }
8728 return 0;
8729}
8730
8731int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
8732{
8733 int ret = 0;
8734 u64 reg;
8735
8736 if (dd->icode != ICODE_RTL_SILICON) {
8737 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
8738 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
8739 __func__);
8740 return -EINVAL;
8741 }
8742 reg = read_csr(dd, ASIC_STS_THERM);
8743 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
8744 ASIC_STS_THERM_CURR_TEMP_MASK);
8745 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
8746 ASIC_STS_THERM_LO_TEMP_MASK);
8747 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
8748 ASIC_STS_THERM_HI_TEMP_MASK);
8749 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
8750 ASIC_STS_THERM_CRIT_TEMP_MASK);
8751 /* triggers is a 3-bit value - 1 bit per trigger. */
8752 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
8753
8754 return ret;
8755}
8756
8757/* ========================================================================= */
8758
8759/*
8760 * Enable/disable chip from delivering interrupts.
8761 */
8762void set_intr_state(struct hfi1_devdata *dd, u32 enable)
8763{
8764 int i;
8765
8766 /*
8767 * In HFI, the mask needs to be 1 to allow interrupts.
8768 */
8769 if (enable) {
8770 u64 cce_int_mask;
8771 const int qsfp1_int_smask = QSFP1_INT % 64;
8772 const int qsfp2_int_smask = QSFP2_INT % 64;
8773
8774 /* enable all interrupts */
8775 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8776 write_csr(dd, CCE_INT_MASK + (8*i), ~(u64)0);
8777
8778 /*
8779 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
8780 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
8781 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
8782 * the index of the appropriate CSR in the CCEIntMask CSR array
8783 */
8784 cce_int_mask = read_csr(dd, CCE_INT_MASK +
8785 (8*(QSFP1_INT/64)));
8786 if (dd->hfi1_id) {
8787 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
8788 write_csr(dd, CCE_INT_MASK + (8*(QSFP1_INT/64)),
8789 cce_int_mask);
8790 } else {
8791 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
8792 write_csr(dd, CCE_INT_MASK + (8*(QSFP2_INT/64)),
8793 cce_int_mask);
8794 }
8795 } else {
8796 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8797 write_csr(dd, CCE_INT_MASK + (8*i), 0ull);
8798 }
8799}
8800
8801/*
8802 * Clear all interrupt sources on the chip.
8803 */
8804static void clear_all_interrupts(struct hfi1_devdata *dd)
8805{
8806 int i;
8807
8808 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8809 write_csr(dd, CCE_INT_CLEAR + (8*i), ~(u64)0);
8810
8811 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
8812 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
8813 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
8814 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
8815 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
8816 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
8817 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
8818 for (i = 0; i < dd->chip_send_contexts; i++)
8819 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
8820 for (i = 0; i < dd->chip_sdma_engines; i++)
8821 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
8822
8823 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
8824 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
8825 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
8826}
8827
8828/* Move to pcie.c? */
8829static void disable_intx(struct pci_dev *pdev)
8830{
8831 pci_intx(pdev, 0);
8832}
8833
8834static void clean_up_interrupts(struct hfi1_devdata *dd)
8835{
8836 int i;
8837
8838 /* remove irqs - must happen before disabling/turning off */
8839 if (dd->num_msix_entries) {
8840 /* MSI-X */
8841 struct hfi1_msix_entry *me = dd->msix_entries;
8842
8843 for (i = 0; i < dd->num_msix_entries; i++, me++) {
8844 if (me->arg == NULL) /* => no irq, no affinity */
8845 break;
8846 irq_set_affinity_hint(dd->msix_entries[i].msix.vector,
8847 NULL);
8848 free_irq(me->msix.vector, me->arg);
8849 }
8850 } else {
8851 /* INTx */
8852 if (dd->requested_intx_irq) {
8853 free_irq(dd->pcidev->irq, dd);
8854 dd->requested_intx_irq = 0;
8855 }
8856 }
8857
8858 /* turn off interrupts */
8859 if (dd->num_msix_entries) {
8860 /* MSI-X */
6e5b6131 8861 pci_disable_msix(dd->pcidev);
77241056
MM		 8862 } else {
8863 /* INTx */
8864 disable_intx(dd->pcidev);
8865 }
8866
8867 /* clean structures */
8868 for (i = 0; i < dd->num_msix_entries; i++)
8869 free_cpumask_var(dd->msix_entries[i].mask);
8870 kfree(dd->msix_entries);
8871 dd->msix_entries = NULL;
8872 dd->num_msix_entries = 0;
8873}
8874
8875/*
8876 * Remap the interrupt source from the general handler to the given MSI-X
8877 * interrupt.
8878 */
8879static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
8880{
8881 u64 reg;
8882 int m, n;
8883
8884 /* clear from the handled mask of the general interrupt */
8885 m = isrc / 64;
8886 n = isrc % 64;
8887 dd->gi_mask[m] &= ~((u64)1 << n);
8888
8889 /* direct the chip source to the given MSI-X interrupt */
8890 m = isrc / 8;
8891 n = isrc % 8;
8892 reg = read_csr(dd, CCE_INT_MAP + (8*m));
8893 reg &= ~((u64)0xff << (8*n));
8894 reg |= ((u64)msix_intr & 0xff) << (8*n);
8895 write_csr(dd, CCE_INT_MAP + (8*m), reg);
8896}
8897
8898static void remap_sdma_interrupts(struct hfi1_devdata *dd,
8899 int engine, int msix_intr)
8900{
8901 /*
8902 * SDMA engine interrupt sources grouped by type, rather than
8903 * engine. Per-engine interrupts are as follows:
8904 * SDMA
8905 * SDMAProgress
8906 * SDMAIdle
8907 */
8908 remap_intr(dd, IS_SDMA_START + 0*TXE_NUM_SDMA_ENGINES + engine,
8909 msix_intr);
8910 remap_intr(dd, IS_SDMA_START + 1*TXE_NUM_SDMA_ENGINES + engine,
8911 msix_intr);
8912 remap_intr(dd, IS_SDMA_START + 2*TXE_NUM_SDMA_ENGINES + engine,
8913 msix_intr);
8914}
8915
77241056
MM		 8916static int request_intx_irq(struct hfi1_devdata *dd)
8917{
8918 int ret;
8919
9805071e
JJ		 8920 snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
8921 dd->unit);
77241056
MM		 8922 ret = request_irq(dd->pcidev->irq, general_interrupt,
8923 IRQF_SHARED, dd->intx_name, dd);
8924 if (ret)
8925 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
8926 ret);
8927 else
8928 dd->requested_intx_irq = 1;
8929 return ret;
8930}
8931
8932static int request_msix_irqs(struct hfi1_devdata *dd)
8933{
8934 const struct cpumask *local_mask;
8935 cpumask_var_t def, rcv;
8936 bool def_ret, rcv_ret;
8937 int first_general, last_general;
8938 int first_sdma, last_sdma;
8939 int first_rx, last_rx;
82c2611d 8940 int first_cpu, curr_cpu;
77241056
MM		 8941 int rcv_cpu, sdma_cpu;
8942 int i, ret = 0, possible;
8943 int ht;
8944
8945 /* calculate the ranges we are going to use */
8946 first_general = 0;
8947 first_sdma = last_general = first_general + 1;
8948 first_rx = last_sdma = first_sdma + dd->num_sdma;
8949 last_rx = first_rx + dd->n_krcv_queues;
8950
8951 /*
8952 * Interrupt affinity.
8953 *
8954 * non-rcv avail gets a default mask that
8955 * starts as possible cpus with threads reset
8956 * and each rcv avail reset.
8957 *
8958 * rcv avail gets node relative 1 wrapping back
8959 * to the node relative 1 as necessary.
8960 *
8961 */
8962 local_mask = cpumask_of_pcibus(dd->pcidev->bus);
8963 /* if first cpu is invalid, use NUMA 0 */
8964 if (cpumask_first(local_mask) >= nr_cpu_ids)
8965 local_mask = topology_core_cpumask(0);
8966
8967 def_ret = zalloc_cpumask_var(&def, GFP_KERNEL);
8968 rcv_ret = zalloc_cpumask_var(&rcv, GFP_KERNEL);
8969 if (!def_ret || !rcv_ret)
8970 goto bail;
8971 /* use local mask as default */
8972 cpumask_copy(def, local_mask);
8973 possible = cpumask_weight(def);
8974 /* disarm threads from default */
8975 ht = cpumask_weight(
8976 topology_sibling_cpumask(cpumask_first(local_mask)));
8977 for (i = possible/ht; i < possible; i++)
8978 cpumask_clear_cpu(i, def);
77241056
MM		 8979 /* def now has full cores on chosen node*/
8980 first_cpu = cpumask_first(def);
8981 if (nr_cpu_ids >= first_cpu)
8982 first_cpu++;
82c2611d 8983 curr_cpu = first_cpu;
77241056 8984
82c2611d
NV		 8985 /* One context is reserved as control context */
8986 for (i = first_cpu; i < dd->n_krcv_queues + first_cpu - 1; i++) {
77241056
MM		 8987 cpumask_clear_cpu(curr_cpu, def);
8988 cpumask_set_cpu(curr_cpu, rcv);
82c2611d
NV		 8989 curr_cpu = cpumask_next(curr_cpu, def);
8990 if (curr_cpu >= nr_cpu_ids)
8991 break;
77241056
MM		 8992 }
8993 /* def mask has non-rcv, rcv has recv mask */
8994 rcv_cpu = cpumask_first(rcv);
8995 sdma_cpu = cpumask_first(def);
8996
8997 /*
8998 * Sanity check - the code expects all SDMA chip source
8999 * interrupts to be in the same CSR, starting at bit 0. Verify
9000 * that this is true by checking the bit location of the start.
9001 */
9002 BUILD_BUG_ON(IS_SDMA_START % 64);
9003
9004 for (i = 0; i < dd->num_msix_entries; i++) {
9005 struct hfi1_msix_entry *me = &dd->msix_entries[i];
9006 const char *err_info;
9007 irq_handler_t handler;
f4f30031 9008 irq_handler_t thread = NULL;
77241056
MM		 9009 void *arg;
9010 int idx;
9011 struct hfi1_ctxtdata *rcd = NULL;
9012 struct sdma_engine *sde = NULL;
9013
9014 /* obtain the arguments to request_irq */
9015 if (first_general <= i && i < last_general) {
9016 idx = i - first_general;
9017 handler = general_interrupt;
9018 arg = dd;
9019 snprintf(me->name, sizeof(me->name),
9805071e 9020 DRIVER_NAME "_%d", dd->unit);
77241056
MM		 9021 err_info = "general";
9022 } else if (first_sdma <= i && i < last_sdma) {
9023 idx = i - first_sdma;
9024 sde = &dd->per_sdma[idx];
9025 handler = sdma_interrupt;
9026 arg = sde;
9027 snprintf(me->name, sizeof(me->name),
9805071e 9028 DRIVER_NAME "_%d sdma%d", dd->unit, idx);
77241056
MM		 9029 err_info = "sdma";
9030 remap_sdma_interrupts(dd, idx, i);
9031 } else if (first_rx <= i && i < last_rx) {
9032 idx = i - first_rx;
9033 rcd = dd->rcd[idx];
9034 /* no interrupt if no rcd */
9035 if (!rcd)
9036 continue;
9037 /*
9038 * Set the interrupt register and mask for this
9039 * context's interrupt.
9040 */
9041 rcd->ireg = (IS_RCVAVAIL_START+idx) / 64;
9042 rcd->imask = ((u64)1) <<
9043 ((IS_RCVAVAIL_START+idx) % 64);
9044 handler = receive_context_interrupt;
f4f30031 9045 thread = receive_context_thread;
77241056
MM		 9046 arg = rcd;
9047 snprintf(me->name, sizeof(me->name),
9805071e 9048 DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
77241056 9049 err_info = "receive context";
66c0933b 9050 remap_intr(dd, IS_RCVAVAIL_START + idx, i);
77241056
MM		 9051 } else {
9052 /* not in our expected range - complain, then
9053 ignore it */
9054 dd_dev_err(dd,
9055 "Unexpected extra MSI-X interrupt %d\n", i);
9056 continue;
9057 }
9058 /* no argument, no interrupt */
9059 if (arg == NULL)
9060 continue;
9061 /* make sure the name is terminated */
9062 me->name[sizeof(me->name)-1] = 0;
9063
f4f30031
DL		 9064 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
9065 me->name, arg);
77241056
MM		 9066 if (ret) {
9067 dd_dev_err(dd,
9068 "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
9069 err_info, me->msix.vector, idx, ret);
9070 return ret;
9071 }
9072 /*
9073 * assign arg after request_irq call, so it will be
9074 * cleaned up
9075 */
9076 me->arg = arg;
9077
9078 if (!zalloc_cpumask_var(
9079 &dd->msix_entries[i].mask,
9080 GFP_KERNEL))
9081 goto bail;
9082 if (handler == sdma_interrupt) {
9083 dd_dev_info(dd, "sdma engine %d cpu %d\n",
9084 sde->this_idx, sdma_cpu);
0a226edd 9085 sde->cpu = sdma_cpu;
77241056
MM		 9086 cpumask_set_cpu(sdma_cpu, dd->msix_entries[i].mask);
9087 sdma_cpu = cpumask_next(sdma_cpu, def);
9088 if (sdma_cpu >= nr_cpu_ids)
9089 sdma_cpu = cpumask_first(def);
9090 } else if (handler == receive_context_interrupt) {
82c2611d
NV		 9091 dd_dev_info(dd, "rcv ctxt %d cpu %d\n", rcd->ctxt,
9092 (rcd->ctxt == HFI1_CTRL_CTXT) ?
9093 cpumask_first(def) : rcv_cpu);
9094 if (rcd->ctxt == HFI1_CTRL_CTXT) {
9095 /* map to first default */
9096 cpumask_set_cpu(cpumask_first(def),
9097 dd->msix_entries[i].mask);
9098 } else {
9099 cpumask_set_cpu(rcv_cpu,
9100 dd->msix_entries[i].mask);
9101 rcv_cpu = cpumask_next(rcv_cpu, rcv);
9102 if (rcv_cpu >= nr_cpu_ids)
9103 rcv_cpu = cpumask_first(rcv);
9104 }
77241056
MM		 9105 } else {
9106 /* otherwise first def */
9107 dd_dev_info(dd, "%s cpu %d\n",
9108 err_info, cpumask_first(def));
9109 cpumask_set_cpu(
9110 cpumask_first(def), dd->msix_entries[i].mask);
9111 }
9112 irq_set_affinity_hint(
9113 dd->msix_entries[i].msix.vector,
9114 dd->msix_entries[i].mask);
9115 }
9116
9117out:
9118 free_cpumask_var(def);
9119 free_cpumask_var(rcv);
9120 return ret;
9121bail:
9122 ret = -ENOMEM;
9123 goto out;
9124}
9125
9126/*
9127 * Set the general handler to accept all interrupts, remap all
9128 * chip interrupts back to MSI-X 0.
9129 */
9130static void reset_interrupts(struct hfi1_devdata *dd)
9131{
9132 int i;
9133
9134 /* all interrupts handled by the general handler */
9135 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
9136 dd->gi_mask[i] = ~(u64)0;
9137
9138 /* all chip interrupts map to MSI-X 0 */
9139 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9140 write_csr(dd, CCE_INT_MAP + (8*i), 0);
9141}
9142
9143static int set_up_interrupts(struct hfi1_devdata *dd)
9144{
9145 struct hfi1_msix_entry *entries;
9146 u32 total, request;
9147 int i, ret;
9148 int single_interrupt = 0; /* we expect to have all the interrupts */
9149
9150 /*
9151 * Interrupt count:
9152 * 1 general, "slow path" interrupt (includes the SDMA engines
9153 * slow source, SDMACleanupDone)
9154 * N interrupts - one per used SDMA engine
9155 * M interrupt - one per kernel receive context
9156 */
9157 total = 1 + dd->num_sdma + dd->n_krcv_queues;
9158
9159 entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
9160 if (!entries) {
77241056
MM		 9161 ret = -ENOMEM;
9162 goto fail;
9163 }
9164 /* 1-1 MSI-X entry assignment */
9165 for (i = 0; i < total; i++)
9166 entries[i].msix.entry = i;
9167
9168 /* ask for MSI-X interrupts */
9169 request = total;
9170 request_msix(dd, &request, entries);
9171
9172 if (request == 0) {
9173 /* using INTx */
9174 /* dd->num_msix_entries already zero */
9175 kfree(entries);
9176 single_interrupt = 1;
9177 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
9178 } else {
9179 /* using MSI-X */
9180 dd->num_msix_entries = request;
9181 dd->msix_entries = entries;
9182
9183 if (request != total) {
9184 /* using MSI-X, with reduced interrupts */
9185 dd_dev_err(
9186 dd,
9187 "cannot handle reduced interrupt case, want %u, got %u\n",
9188 total, request);
9189 ret = -EINVAL;
9190 goto fail;
9191 }
9192 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
9193 }
9194
9195 /* mask all interrupts */
9196 set_intr_state(dd, 0);
9197 /* clear all pending interrupts */
9198 clear_all_interrupts(dd);
9199
9200 /* reset general handler mask, chip MSI-X mappings */
9201 reset_interrupts(dd);
9202
9203 if (single_interrupt)
9204 ret = request_intx_irq(dd);
9205 else
9206 ret = request_msix_irqs(dd);
9207 if (ret)
9208 goto fail;
9209
9210 return 0;
9211
9212fail:
9213 clean_up_interrupts(dd);
9214 return ret;
9215}
9216
9217/*
9218 * Set up context values in dd. Sets:
9219 *
9220 * num_rcv_contexts - number of contexts being used
9221 * n_krcv_queues - number of kernel contexts
9222 * first_user_ctxt - first non-kernel context in array of contexts
9223 * freectxts - number of free user contexts
9224 * num_send_contexts - number of PIO send contexts being used
9225 */
9226static int set_up_context_variables(struct hfi1_devdata *dd)
9227{
9228 int num_kernel_contexts;
9229 int num_user_contexts;
9230 int total_contexts;
9231 int ret;
9232 unsigned ngroups;
9233
9234 /*
9235 * Kernel contexts: (to be fixed later):
9236 * - min or 2 or 1 context/numa
82c2611d
NV		 9237 * - Context 0 - control context (VL15/multicast/error)
9238 * - Context 1 - default context
77241056
MM		 9239 */
9240 if (n_krcvqs)
82c2611d
NV		 9241 /*
9242 * Don't count context 0 in n_krcvqs since
9243 * is isn't used for normal verbs traffic.
9244 *
9245 * krcvqs will reflect number of kernel
9246 * receive contexts above 0.
9247 */
9248 num_kernel_contexts = n_krcvqs + MIN_KERNEL_KCTXTS - 1;
77241056
MM		 9249 else
9250 num_kernel_contexts = num_online_nodes();
9251 num_kernel_contexts =
9252 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
9253 /*
9254 * Every kernel receive context needs an ACK send context.
9255 * one send context is allocated for each VL{0-7} and VL15
9256 */
9257 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
9258 dd_dev_err(dd,
9259 "Reducing # kernel rcv contexts to: %d, from %d\n",
9260 (int)(dd->chip_send_contexts - num_vls - 1),
9261 (int)num_kernel_contexts);
9262 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
9263 }
9264 /*
9265 * User contexts: (to be fixed later)
9266 * - set to num_rcv_contexts if non-zero
9267 * - default to 1 user context per CPU
9268 */
9269 if (num_rcv_contexts)
9270 num_user_contexts = num_rcv_contexts;
9271 else
9272 num_user_contexts = num_online_cpus();
9273
9274 total_contexts = num_kernel_contexts + num_user_contexts;
9275
9276 /*
9277 * Adjust the counts given a global max.
9278 */
9279 if (total_contexts > dd->chip_rcv_contexts) {
9280 dd_dev_err(dd,
9281 "Reducing # user receive contexts to: %d, from %d\n",
9282 (int)(dd->chip_rcv_contexts - num_kernel_contexts),
9283 (int)num_user_contexts);
9284 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
9285 /* recalculate */
9286 total_contexts = num_kernel_contexts + num_user_contexts;
9287 }
9288
9289 /* the first N are kernel contexts, the rest are user contexts */
9290 dd->num_rcv_contexts = total_contexts;
9291 dd->n_krcv_queues = num_kernel_contexts;
9292 dd->first_user_ctxt = num_kernel_contexts;
9293 dd->freectxts = num_user_contexts;
9294 dd_dev_info(dd,
9295 "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
9296 (int)dd->chip_rcv_contexts,
9297 (int)dd->num_rcv_contexts,
9298 (int)dd->n_krcv_queues,
9299 (int)dd->num_rcv_contexts - dd->n_krcv_queues);
9300
9301 /*
9302 * Receive array allocation:
9303 * All RcvArray entries are divided into groups of 8. This
9304 * is required by the hardware and will speed up writes to
9305 * consecutive entries by using write-combining of the entire
9306 * cacheline.
9307 *
9308 * The number of groups are evenly divided among all contexts.
9309 * any left over groups will be given to the first N user
9310 * contexts.
9311 */
9312 dd->rcv_entries.group_size = RCV_INCREMENT;
9313 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
9314 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
9315 dd->rcv_entries.nctxt_extra = ngroups -
9316 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
9317 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
9318 dd->rcv_entries.ngroups,
9319 dd->rcv_entries.nctxt_extra);
9320 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
9321 MAX_EAGER_ENTRIES * 2) {
9322 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
9323 dd->rcv_entries.group_size;
9324 dd_dev_info(dd,
9325 "RcvArray group count too high, change to %u\n",
9326 dd->rcv_entries.ngroups);
9327 dd->rcv_entries.nctxt_extra = 0;
9328 }
9329 /*
9330 * PIO send contexts
9331 */
9332 ret = init_sc_pools_and_sizes(dd);
9333 if (ret >= 0) { /* success */
9334 dd->num_send_contexts = ret;
9335 dd_dev_info(
9336 dd,
9337 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d)\n",
9338 dd->chip_send_contexts,
9339 dd->num_send_contexts,
9340 dd->sc_sizes[SC_KERNEL].count,
9341 dd->sc_sizes[SC_ACK].count,
9342 dd->sc_sizes[SC_USER].count);
9343 ret = 0; /* success */
9344 }
9345
9346 return ret;
9347}
9348
9349/*
9350 * Set the device/port partition key table. The MAD code
9351 * will ensure that, at least, the partial management
9352 * partition key is present in the table.
9353 */
9354static void set_partition_keys(struct hfi1_pportdata *ppd)
9355{
9356 struct hfi1_devdata *dd = ppd->dd;
9357 u64 reg = 0;
9358 int i;
9359
9360 dd_dev_info(dd, "Setting partition keys\n");
9361 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
9362 reg |= (ppd->pkeys[i] &
9363 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
9364 ((i % 4) *
9365 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
9366 /* Each register holds 4 PKey values. */
9367 if ((i % 4) == 3) {
9368 write_csr(dd, RCV_PARTITION_KEY +
9369 ((i - 3) * 2), reg);
9370 reg = 0;
9371 }
9372 }
9373
9374 /* Always enable HW pkeys check when pkeys table is set */
9375 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
9376}
9377
9378/*
9379 * These CSRs and memories are uninitialized on reset and must be
9380 * written before reading to set the ECC/parity bits.
9381 *
9382 * NOTE: All user context CSRs that are not mmaped write-only
9383 * (e.g. the TID flows) must be initialized even if the driver never
9384 * reads them.
9385 */
9386static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
9387{
9388 int i, j;
9389
9390 /* CceIntMap */
9391 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9392 write_csr(dd, CCE_INT_MAP+(8*i), 0);
9393
9394 /* SendCtxtCreditReturnAddr */
9395 for (i = 0; i < dd->chip_send_contexts; i++)
9396 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
9397
9398 /* PIO Send buffers */
9399 /* SDMA Send buffers */
9400 /* These are not normally read, and (presently) have no method
9401 to be read, so are not pre-initialized */
9402
9403 /* RcvHdrAddr */
9404 /* RcvHdrTailAddr */
9405 /* RcvTidFlowTable */
9406 for (i = 0; i < dd->chip_rcv_contexts; i++) {
9407 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
9408 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
9409 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
9410 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE+(8*j), 0);
9411 }
9412
9413 /* RcvArray */
9414 for (i = 0; i < dd->chip_rcv_array_count; i++)
9415 write_csr(dd, RCV_ARRAY + (8*i),
9416 RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
9417
9418 /* RcvQPMapTable */
9419 for (i = 0; i < 32; i++)
9420 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
9421}
9422
9423/*
9424 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
9425 */
9426static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
9427 u64 ctrl_bits)
9428{
9429 unsigned long timeout;
9430 u64 reg;
9431
9432 /* is the condition present? */
9433 reg = read_csr(dd, CCE_STATUS);
9434 if ((reg & status_bits) == 0)
9435 return;
9436
9437 /* clear the condition */
9438 write_csr(dd, CCE_CTRL, ctrl_bits);
9439
9440 /* wait for the condition to clear */
9441 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
9442 while (1) {
9443 reg = read_csr(dd, CCE_STATUS);
9444 if ((reg & status_bits) == 0)
9445 return;
9446 if (time_after(jiffies, timeout)) {
9447 dd_dev_err(dd,
9448 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
9449 status_bits, reg & status_bits);
9450 return;
9451 }
9452 udelay(1);
9453 }
9454}
9455
9456/* set CCE CSRs to chip reset defaults */
9457static void reset_cce_csrs(struct hfi1_devdata *dd)
9458{
9459 int i;
9460
9461 /* CCE_REVISION read-only */
9462 /* CCE_REVISION2 read-only */
9463 /* CCE_CTRL - bits clear automatically */
9464 /* CCE_STATUS read-only, use CceCtrl to clear */
9465 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
9466 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
9467 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
9468 for (i = 0; i < CCE_NUM_SCRATCH; i++)
9469 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
9470 /* CCE_ERR_STATUS read-only */
9471 write_csr(dd, CCE_ERR_MASK, 0);
9472 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
9473 /* CCE_ERR_FORCE leave alone */
9474 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
9475 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
9476 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
9477 /* CCE_PCIE_CTRL leave alone */
9478 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
9479 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
9480 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
9481 CCE_MSIX_TABLE_UPPER_RESETCSR);
9482 }
9483 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
9484 /* CCE_MSIX_PBA read-only */
9485 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
9486 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
9487 }
9488 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
9489 write_csr(dd, CCE_INT_MAP, 0);
9490 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
9491 /* CCE_INT_STATUS read-only */
9492 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
9493 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
9494 /* CCE_INT_FORCE leave alone */
9495 /* CCE_INT_BLOCKED read-only */
9496 }
9497 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
9498 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
9499}
9500
9501/* set ASIC CSRs to chip reset defaults */
9502static void reset_asic_csrs(struct hfi1_devdata *dd)
9503{
77241056
MM		 9504 int i;
9505
9506 /*
9507 * If the HFIs are shared between separate nodes or VMs,
9508 * then more will need to be done here. One idea is a module
9509 * parameter that returns early, letting the first power-on or
9510 * a known first load do the reset and blocking all others.
9511 */
9512
7c03ed85
EH		 9513 if (!(dd->flags & HFI1_DO_INIT_ASIC))
9514 return;
77241056
MM		 9515
9516 if (dd->icode != ICODE_FPGA_EMULATION) {
9517 /* emulation does not have an SBus - leave these alone */
9518 /*
9519 * All writes to ASIC_CFG_SBUS_REQUEST do something.
9520 * Notes:
9521 * o The reset is not zero if aimed at the core. See the
9522 * SBus documentation for details.
9523 * o If the SBus firmware has been updated (e.g. by the BIOS),
9524 * will the reset revert that?
9525 */
9526 /* ASIC_CFG_SBUS_REQUEST leave alone */
9527 write_csr(dd, ASIC_CFG_SBUS_EXECUTE, 0);
9528 }
9529 /* ASIC_SBUS_RESULT read-only */
9530 write_csr(dd, ASIC_STS_SBUS_COUNTERS, 0);
9531 for (i = 0; i < ASIC_NUM_SCRATCH; i++)
9532 write_csr(dd, ASIC_CFG_SCRATCH + (8 * i), 0);
9533 write_csr(dd, ASIC_CFG_MUTEX, 0); /* this will clear it */
7c03ed85
EH		 9534
9535 /* We might want to retain this state across FLR if we ever use it */
77241056 9536 write_csr(dd, ASIC_CFG_DRV_STR, 0);
7c03ed85 9537
4ef98989 9538 /* ASIC_CFG_THERM_POLL_EN leave alone */
77241056
MM		 9539 /* ASIC_STS_THERM read-only */
9540 /* ASIC_CFG_RESET leave alone */
9541
9542 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, 0);
9543 /* ASIC_PCIE_SD_HOST_STATUS read-only */
9544 write_csr(dd, ASIC_PCIE_SD_INTRPT_DATA_CODE, 0);
9545 write_csr(dd, ASIC_PCIE_SD_INTRPT_ENABLE, 0);
9546 /* ASIC_PCIE_SD_INTRPT_PROGRESS read-only */
9547 write_csr(dd, ASIC_PCIE_SD_INTRPT_STATUS, ~0ull); /* clear */
9548 /* ASIC_HFI0_PCIE_SD_INTRPT_RSPD_DATA read-only */
9549 /* ASIC_HFI1_PCIE_SD_INTRPT_RSPD_DATA read-only */
9550 for (i = 0; i < 16; i++)
9551 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (8 * i), 0);
9552
9553 /* ASIC_GPIO_IN read-only */
9554 write_csr(dd, ASIC_GPIO_OE, 0);
9555 write_csr(dd, ASIC_GPIO_INVERT, 0);
9556 write_csr(dd, ASIC_GPIO_OUT, 0);
9557 write_csr(dd, ASIC_GPIO_MASK, 0);
9558 /* ASIC_GPIO_STATUS read-only */
9559 write_csr(dd, ASIC_GPIO_CLEAR, ~0ull);
9560 /* ASIC_GPIO_FORCE leave alone */
9561
9562 /* ASIC_QSFP1_IN read-only */
9563 write_csr(dd, ASIC_QSFP1_OE, 0);
9564 write_csr(dd, ASIC_QSFP1_INVERT, 0);
9565 write_csr(dd, ASIC_QSFP1_OUT, 0);
9566 write_csr(dd, ASIC_QSFP1_MASK, 0);
9567 /* ASIC_QSFP1_STATUS read-only */
9568 write_csr(dd, ASIC_QSFP1_CLEAR, ~0ull);
9569 /* ASIC_QSFP1_FORCE leave alone */
9570
9571 /* ASIC_QSFP2_IN read-only */
9572 write_csr(dd, ASIC_QSFP2_OE, 0);
9573 write_csr(dd, ASIC_QSFP2_INVERT, 0);
9574 write_csr(dd, ASIC_QSFP2_OUT, 0);
9575 write_csr(dd, ASIC_QSFP2_MASK, 0);
9576 /* ASIC_QSFP2_STATUS read-only */
9577 write_csr(dd, ASIC_QSFP2_CLEAR, ~0ull);
9578 /* ASIC_QSFP2_FORCE leave alone */
9579
9580 write_csr(dd, ASIC_EEP_CTL_STAT, ASIC_EEP_CTL_STAT_RESETCSR);
9581 /* this also writes a NOP command, clearing paging mode */
9582 write_csr(dd, ASIC_EEP_ADDR_CMD, 0);
9583 write_csr(dd, ASIC_EEP_DATA, 0);
77241056
MM		 9584}
9585
9586/* set MISC CSRs to chip reset defaults */
9587static void reset_misc_csrs(struct hfi1_devdata *dd)
9588{
9589 int i;
9590
9591 for (i = 0; i < 32; i++) {
9592 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
9593 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
9594 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
9595 }
9596 /* MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
9597 only be written 128-byte chunks */
9598 /* init RSA engine to clear lingering errors */
9599 write_csr(dd, MISC_CFG_RSA_CMD, 1);
9600 write_csr(dd, MISC_CFG_RSA_MU, 0);
9601 write_csr(dd, MISC_CFG_FW_CTRL, 0);
9602 /* MISC_STS_8051_DIGEST read-only */
9603 /* MISC_STS_SBM_DIGEST read-only */
9604 /* MISC_STS_PCIE_DIGEST read-only */
9605 /* MISC_STS_FAB_DIGEST read-only */
9606 /* MISC_ERR_STATUS read-only */
9607 write_csr(dd, MISC_ERR_MASK, 0);
9608 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
9609 /* MISC_ERR_FORCE leave alone */
9610}
9611
9612/* set TXE CSRs to chip reset defaults */
9613static void reset_txe_csrs(struct hfi1_devdata *dd)
9614{
9615 int i;
9616
9617 /*
9618 * TXE Kernel CSRs
9619 */
9620 write_csr(dd, SEND_CTRL, 0);
9621 __cm_reset(dd, 0); /* reset CM internal state */
9622 /* SEND_CONTEXTS read-only */
9623 /* SEND_DMA_ENGINES read-only */
9624 /* SEND_PIO_MEM_SIZE read-only */
9625 /* SEND_DMA_MEM_SIZE read-only */
9626 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
9627 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
9628 /* SEND_PIO_ERR_STATUS read-only */
9629 write_csr(dd, SEND_PIO_ERR_MASK, 0);
9630 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
9631 /* SEND_PIO_ERR_FORCE leave alone */
9632 /* SEND_DMA_ERR_STATUS read-only */
9633 write_csr(dd, SEND_DMA_ERR_MASK, 0);
9634 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
9635 /* SEND_DMA_ERR_FORCE leave alone */
9636 /* SEND_EGRESS_ERR_STATUS read-only */
9637 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
9638 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
9639 /* SEND_EGRESS_ERR_FORCE leave alone */
9640 write_csr(dd, SEND_BTH_QP, 0);
9641 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
9642 write_csr(dd, SEND_SC2VLT0, 0);
9643 write_csr(dd, SEND_SC2VLT1, 0);
9644 write_csr(dd, SEND_SC2VLT2, 0);
9645 write_csr(dd, SEND_SC2VLT3, 0);
9646 write_csr(dd, SEND_LEN_CHECK0, 0);
9647 write_csr(dd, SEND_LEN_CHECK1, 0);
9648 /* SEND_ERR_STATUS read-only */
9649 write_csr(dd, SEND_ERR_MASK, 0);
9650 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
9651 /* SEND_ERR_FORCE read-only */
9652 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
9653 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8*i), 0);
9654 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
9655 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8*i), 0);
9656 for (i = 0; i < dd->chip_send_contexts/NUM_CONTEXTS_PER_SET; i++)
9657 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8*i), 0);
9658 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
9659 write_csr(dd, SEND_COUNTER_ARRAY32 + (8*i), 0);
9660 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
9661 write_csr(dd, SEND_COUNTER_ARRAY64 + (8*i), 0);
9662 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
9663 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
9664 SEND_CM_GLOBAL_CREDIT_RESETCSR);
9665 /* SEND_CM_CREDIT_USED_STATUS read-only */
9666 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
9667 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
9668 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
9669 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
9670 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
9671 for (i = 0; i < TXE_NUM_DATA_VL; i++)
9672 write_csr(dd, SEND_CM_CREDIT_VL + (8*i), 0);
9673 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
9674 /* SEND_CM_CREDIT_USED_VL read-only */
9675 /* SEND_CM_CREDIT_USED_VL15 read-only */
9676 /* SEND_EGRESS_CTXT_STATUS read-only */
9677 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
9678 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
9679 /* SEND_EGRESS_ERR_INFO read-only */
9680 /* SEND_EGRESS_ERR_SOURCE read-only */
9681
9682 /*
9683 * TXE Per-Context CSRs
9684 */
9685 for (i = 0; i < dd->chip_send_contexts; i++) {
9686 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
9687 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
9688 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
9689 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
9690 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
9691 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
9692 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
9693 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
9694 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
9695 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
9696 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
9697 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
9698 }
9699
9700 /*
9701 * TXE Per-SDMA CSRs
9702 */
9703 for (i = 0; i < dd->chip_sdma_engines; i++) {
9704 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
9705 /* SEND_DMA_STATUS read-only */
9706 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
9707 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
9708 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
9709 /* SEND_DMA_HEAD read-only */
9710 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
9711 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
9712 /* SEND_DMA_IDLE_CNT read-only */
9713 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
9714 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
9715 /* SEND_DMA_DESC_FETCHED_CNT read-only */
9716 /* SEND_DMA_ENG_ERR_STATUS read-only */
9717 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
9718 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
9719 /* SEND_DMA_ENG_ERR_FORCE leave alone */
9720 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
9721 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
9722 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
9723 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
9724 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
9725 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
9726 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
9727 }
9728}
9729
9730/*
9731 * Expect on entry:
9732 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
9733 */
9734static void init_rbufs(struct hfi1_devdata *dd)
9735{
9736 u64 reg;
9737 int count;
9738
9739 /*
9740 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
9741 * clear.
9742 */
9743 count = 0;
9744 while (1) {
9745 reg = read_csr(dd, RCV_STATUS);
9746 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
9747 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
9748 break;
9749 /*
9750 * Give up after 1ms - maximum wait time.
9751 *
9752 * RBuf size is 148KiB. Slowest possible is PCIe Gen1 x1 at
9753 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
9754 * 148 KB / (66% * 250MB/s) = 920us
9755 */
9756 if (count++ > 500) {
9757 dd_dev_err(dd,
9758 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
9759 __func__, reg);
9760 break;
9761 }
9762 udelay(2); /* do not busy-wait the CSR */
9763 }
9764
9765 /* start the init - expect RcvCtrl to be 0 */
9766 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
9767
9768 /*
9769 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
9770 * period after the write before RcvStatus.RxRbufInitDone is valid.
9771 * The delay in the first run through the loop below is sufficient and
9772 * required before the first read of RcvStatus.RxRbufInintDone.
9773 */
9774 read_csr(dd, RCV_CTRL);
9775
9776 /* wait for the init to finish */
9777 count = 0;
9778 while (1) {
9779 /* delay is required first time through - see above */
9780 udelay(2); /* do not busy-wait the CSR */
9781 reg = read_csr(dd, RCV_STATUS);
9782 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
9783 break;
9784
9785 /* give up after 100us - slowest possible at 33MHz is 73us */
9786 if (count++ > 50) {
9787 dd_dev_err(dd,
9788 "%s: RcvStatus.RxRbufInit not set, continuing\n",
9789 __func__);
9790 break;
9791 }
9792 }
9793}
9794
9795/* set RXE CSRs to chip reset defaults */
9796static void reset_rxe_csrs(struct hfi1_devdata *dd)
9797{
9798 int i, j;
9799
9800 /*
9801 * RXE Kernel CSRs
9802 */
9803 write_csr(dd, RCV_CTRL, 0);
9804 init_rbufs(dd);
9805 /* RCV_STATUS read-only */
9806 /* RCV_CONTEXTS read-only */
9807 /* RCV_ARRAY_CNT read-only */
9808 /* RCV_BUF_SIZE read-only */
9809 write_csr(dd, RCV_BTH_QP, 0);
9810 write_csr(dd, RCV_MULTICAST, 0);
9811 write_csr(dd, RCV_BYPASS, 0);
9812 write_csr(dd, RCV_VL15, 0);
9813 /* this is a clear-down */
9814 write_csr(dd, RCV_ERR_INFO,
9815 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
9816 /* RCV_ERR_STATUS read-only */
9817 write_csr(dd, RCV_ERR_MASK, 0);
9818 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
9819 /* RCV_ERR_FORCE leave alone */
9820 for (i = 0; i < 32; i++)
9821 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
9822 for (i = 0; i < 4; i++)
9823 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
9824 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
9825 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
9826 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
9827 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
9828 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
9829 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
9830 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
9831 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
9832 }
9833 for (i = 0; i < 32; i++)
9834 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
9835
9836 /*
9837 * RXE Kernel and User Per-Context CSRs
9838 */
9839 for (i = 0; i < dd->chip_rcv_contexts; i++) {
9840 /* kernel */
9841 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
9842 /* RCV_CTXT_STATUS read-only */
9843 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
9844 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
9845 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
9846 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
9847 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
9848 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
9849 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
9850 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
9851 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
9852 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
9853
9854 /* user */
9855 /* RCV_HDR_TAIL read-only */
9856 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
9857 /* RCV_EGR_INDEX_TAIL read-only */
9858 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
9859 /* RCV_EGR_OFFSET_TAIL read-only */
9860 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
9861 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j),
9862 0);
9863 }
9864 }
9865}
9866
9867/*
9868 * Set sc2vl tables.
9869 *
9870 * They power on to zeros, so to avoid send context errors
9871 * they need to be set:
9872 *
9873 * SC 0-7 -> VL 0-7 (respectively)
9874 * SC 15 -> VL 15
9875 * otherwise
9876 * -> VL 0
9877 */
9878static void init_sc2vl_tables(struct hfi1_devdata *dd)
9879{
9880 int i;
9881 /* init per architecture spec, constrained by hardware capability */
9882
9883 /* HFI maps sent packets */
9884 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
9885 0,
9886 0, 0, 1, 1,
9887 2, 2, 3, 3,
9888 4, 4, 5, 5,
9889 6, 6, 7, 7));
9890 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
9891 1,
9892 8, 0, 9, 0,
9893 10, 0, 11, 0,
9894 12, 0, 13, 0,
9895 14, 0, 15, 15));
9896 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
9897 2,
9898 16, 0, 17, 0,
9899 18, 0, 19, 0,
9900 20, 0, 21, 0,
9901 22, 0, 23, 0));
9902 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
9903 3,
9904 24, 0, 25, 0,
9905 26, 0, 27, 0,
9906 28, 0, 29, 0,
9907 30, 0, 31, 0));
9908
9909 /* DC maps received packets */
9910 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
9911 15_0,
9912 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
9913 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
9914 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
9915 31_16,
9916 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
9917 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
9918
9919 /* initialize the cached sc2vl values consistently with h/w */
9920 for (i = 0; i < 32; i++) {
9921 if (i < 8 || i == 15)
9922 *((u8 *)(dd->sc2vl) + i) = (u8)i;
9923 else
9924 *((u8 *)(dd->sc2vl) + i) = 0;
9925 }
9926}
9927
9928/*
9929 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
9930 * depend on the chip going through a power-on reset - a driver may be loaded
9931 * and unloaded many times.
9932 *
9933 * Do not write any CSR values to the chip in this routine - there may be
9934 * a reset following the (possible) FLR in this routine.
9935 *
9936 */
9937static void init_chip(struct hfi1_devdata *dd)
9938{
9939 int i;
9940
9941 /*
9942 * Put the HFI CSRs in a known state.
9943 * Combine this with a DC reset.
9944 *
9945 * Stop the device from doing anything while we do a
9946 * reset. We know there are no other active users of
9947 * the device since we are now in charge. Turn off
9948 * off all outbound and inbound traffic and make sure
9949 * the device does not generate any interrupts.
9950 */
9951
9952 /* disable send contexts and SDMA engines */
9953 write_csr(dd, SEND_CTRL, 0);
9954 for (i = 0; i < dd->chip_send_contexts; i++)
9955 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
9956 for (i = 0; i < dd->chip_sdma_engines; i++)
9957 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
9958 /* disable port (turn off RXE inbound traffic) and contexts */
9959 write_csr(dd, RCV_CTRL, 0);
9960 for (i = 0; i < dd->chip_rcv_contexts; i++)
9961 write_csr(dd, RCV_CTXT_CTRL, 0);
9962 /* mask all interrupt sources */
9963 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
9964 write_csr(dd, CCE_INT_MASK + (8*i), 0ull);
9965
9966 /*
9967 * DC Reset: do a full DC reset before the register clear.
9968 * A recommended length of time to hold is one CSR read,
9969 * so reread the CceDcCtrl. Then, hold the DC in reset
9970 * across the clear.
9971 */
9972 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
9973 (void) read_csr(dd, CCE_DC_CTRL);
9974
9975 if (use_flr) {
9976 /*
9977 * A FLR will reset the SPC core and part of the PCIe.
9978 * The parts that need to be restored have already been
9979 * saved.
9980 */
9981 dd_dev_info(dd, "Resetting CSRs with FLR\n");
9982
9983 /* do the FLR, the DC reset will remain */
9984 hfi1_pcie_flr(dd);
9985
9986 /* restore command and BARs */
9987 restore_pci_variables(dd);
9988
995deafa 9989 if (is_ax(dd)) {
77241056
MM		 9990 dd_dev_info(dd, "Resetting CSRs with FLR\n");
9991 hfi1_pcie_flr(dd);
9992 restore_pci_variables(dd);
9993 }
9994
7c03ed85 9995 reset_asic_csrs(dd);
77241056
MM		 9996 } else {
9997 dd_dev_info(dd, "Resetting CSRs with writes\n");
9998 reset_cce_csrs(dd);
9999 reset_txe_csrs(dd);
10000 reset_rxe_csrs(dd);
10001 reset_asic_csrs(dd);
10002 reset_misc_csrs(dd);
10003 }
10004 /* clear the DC reset */
10005 write_csr(dd, CCE_DC_CTRL, 0);
7c03ed85 10006
77241056 10007 /* Set the LED off */
995deafa 10008 if (is_ax(dd))
77241056
MM		 10009 setextled(dd, 0);
10010 /*
10011 * Clear the QSFP reset.
72a67ba2 10012 * An FLR enforces a 0 on all out pins. The driver does not touch
77241056 10013 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
72a67ba2 10014 * anything plugged constantly in reset, if it pays attention
77241056 10015 * to RESET_N.
72a67ba2 10016 * Prime examples of this are optical cables. Set all pins high.
77241056
MM		 10017 * I2CCLK and I2CDAT will change per direction, and INT_N and
10018 * MODPRS_N are input only and their value is ignored.
10019 */
72a67ba2
EH		 10020 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
10021 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
77241056
MM		 10022}
10023
10024static void init_early_variables(struct hfi1_devdata *dd)
10025{
10026 int i;
10027
10028 /* assign link credit variables */
10029 dd->vau = CM_VAU;
10030 dd->link_credits = CM_GLOBAL_CREDITS;
995deafa 10031 if (is_ax(dd))
77241056
MM		 10032 dd->link_credits--;
10033 dd->vcu = cu_to_vcu(hfi1_cu);
10034 /* enough room for 8 MAD packets plus header - 17K */
10035 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
10036 if (dd->vl15_init > dd->link_credits)
10037 dd->vl15_init = dd->link_credits;
10038
10039 write_uninitialized_csrs_and_memories(dd);
10040
10041 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10042 for (i = 0; i < dd->num_pports; i++) {
10043 struct hfi1_pportdata *ppd = &dd->pport[i];
10044
10045 set_partition_keys(ppd);
10046 }
10047 init_sc2vl_tables(dd);
10048}
10049
10050static void init_kdeth_qp(struct hfi1_devdata *dd)
10051{
10052 /* user changed the KDETH_QP */
10053 if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
10054 /* out of range or illegal value */
10055 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
10056 kdeth_qp = 0;
10057 }
10058 if (kdeth_qp == 0) /* not set, or failed range check */
10059 kdeth_qp = DEFAULT_KDETH_QP;
10060
10061 write_csr(dd, SEND_BTH_QP,
10062 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK)
10063 << SEND_BTH_QP_KDETH_QP_SHIFT);
10064
10065 write_csr(dd, RCV_BTH_QP,
10066 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK)
10067 << RCV_BTH_QP_KDETH_QP_SHIFT);
10068}
10069
10070/**
10071 * init_qpmap_table
10072 * @dd - device data
10073 * @first_ctxt - first context
10074 * @last_ctxt - first context
10075 *
10076 * This return sets the qpn mapping table that
10077 * is indexed by qpn[8:1].
10078 *
10079 * The routine will round robin the 256 settings
10080 * from first_ctxt to last_ctxt.
10081 *
10082 * The first/last looks ahead to having specialized
10083 * receive contexts for mgmt and bypass. Normal
10084 * verbs traffic will assumed to be on a range
10085 * of receive contexts.
10086 */
10087static void init_qpmap_table(struct hfi1_devdata *dd,
10088 u32 first_ctxt,
10089 u32 last_ctxt)
10090{
10091 u64 reg = 0;
10092 u64 regno = RCV_QP_MAP_TABLE;
10093 int i;
10094 u64 ctxt = first_ctxt;
10095
10096 for (i = 0; i < 256;) {
77241056
MM		 10097 reg |= ctxt << (8 * (i % 8));
10098 i++;
10099 ctxt++;
10100 if (ctxt > last_ctxt)
10101 ctxt = first_ctxt;
10102 if (i % 8 == 0) {
10103 write_csr(dd, regno, reg);
10104 reg = 0;
10105 regno += 8;
10106 }
10107 }
10108 if (i % 8)
10109 write_csr(dd, regno, reg);
10110
10111 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
10112 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
10113}
10114
10115/**
10116 * init_qos - init RX qos
10117 * @dd - device data
10118 * @first_context
10119 *
10120 * This routine initializes Rule 0 and the
10121 * RSM map table to implement qos.
10122 *
10123 * If all of the limit tests succeed,
10124 * qos is applied based on the array
10125 * interpretation of krcvqs where
10126 * entry 0 is VL0.
10127 *
10128 * The number of vl bits (n) and the number of qpn
10129 * bits (m) are computed to feed both the RSM map table
10130 * and the single rule.
10131 *
10132 */
10133static void init_qos(struct hfi1_devdata *dd, u32 first_ctxt)
10134{
10135 u8 max_by_vl = 0;
10136 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
10137 u64 *rsmmap;
10138 u64 reg;
995deafa 10139 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
77241056
MM		 10140
10141 /* validate */
10142 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
10143 num_vls == 1 ||
10144 krcvqsset <= 1)
10145 goto bail;
10146 for (i = 0; i < min_t(unsigned, num_vls, krcvqsset); i++)
10147 if (krcvqs[i] > max_by_vl)
10148 max_by_vl = krcvqs[i];
10149 if (max_by_vl > 32)
10150 goto bail;
10151 qpns_per_vl = __roundup_pow_of_two(max_by_vl);
10152 /* determine bits vl */
10153 n = ilog2(num_vls);
10154 /* determine bits for qpn */
10155 m = ilog2(qpns_per_vl);
10156 if ((m + n) > 7)
10157 goto bail;
10158 if (num_vls * qpns_per_vl > dd->chip_rcv_contexts)
10159 goto bail;
10160 rsmmap = kmalloc_array(NUM_MAP_REGS, sizeof(u64), GFP_KERNEL);
10161 memset(rsmmap, rxcontext, NUM_MAP_REGS * sizeof(u64));
10162 /* init the local copy of the table */
10163 for (i = 0, ctxt = first_ctxt; i < num_vls; i++) {
10164 unsigned tctxt;
10165
10166 for (qpn = 0, tctxt = ctxt;
10167 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
10168 unsigned idx, regoff, regidx;
10169
10170 /* generate index <= 128 */
10171 idx = (qpn << n) ^ i;
10172 regoff = (idx % 8) * 8;
10173 regidx = idx / 8;
10174 reg = rsmmap[regidx];
10175 /* replace 0xff with context number */
10176 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
10177 << regoff);
10178 reg |= (u64)(tctxt++) << regoff;
10179 rsmmap[regidx] = reg;
10180 if (tctxt == ctxt + krcvqs[i])
10181 tctxt = ctxt;
10182 }
10183 ctxt += krcvqs[i];
10184 }
10185 /* flush cached copies to chip */
10186 for (i = 0; i < NUM_MAP_REGS; i++)
10187 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rsmmap[i]);
10188 /* add rule0 */
10189 write_csr(dd, RCV_RSM_CFG /* + (8 * 0) */,
10190 RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_MASK
10191 << RCV_RSM_CFG_ENABLE_OR_CHAIN_RSM0_SHIFT |
10192 2ull << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
10193 write_csr(dd, RCV_RSM_SELECT /* + (8 * 0) */,
10194 LRH_BTH_MATCH_OFFSET
10195 << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
10196 LRH_SC_MATCH_OFFSET << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
10197 LRH_SC_SELECT_OFFSET << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
10198 ((u64)n) << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
10199 QPN_SELECT_OFFSET << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
10200 ((u64)m + (u64)n) << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
10201 write_csr(dd, RCV_RSM_MATCH /* + (8 * 0) */,
10202 LRH_BTH_MASK << RCV_RSM_MATCH_MASK1_SHIFT |
10203 LRH_BTH_VALUE << RCV_RSM_MATCH_VALUE1_SHIFT |
10204 LRH_SC_MASK << RCV_RSM_MATCH_MASK2_SHIFT |
10205 LRH_SC_VALUE << RCV_RSM_MATCH_VALUE2_SHIFT);
10206 /* Enable RSM */
10207 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
10208 kfree(rsmmap);
82c2611d
NV		 10209 /* map everything else to first context */
10210 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, MIN_KERNEL_KCTXTS - 1);
77241056
MM		 10211 dd->qos_shift = n + 1;
10212 return;
10213bail:
10214 dd->qos_shift = 1;
82c2611d 10215 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
77241056
MM		 10216}
10217
10218static void init_rxe(struct hfi1_devdata *dd)
10219{
10220 /* enable all receive errors */
10221 write_csr(dd, RCV_ERR_MASK, ~0ull);
10222 /* setup QPN map table - start where VL15 context leaves off */
10223 init_qos(
10224 dd,
10225 dd->n_krcv_queues > MIN_KERNEL_KCTXTS ? MIN_KERNEL_KCTXTS : 0);
10226 /*
10227 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
10228 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
10229 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
10230 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
10231 * Max_PayLoad_Size set to its minimum of 128.
10232 *
10233 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
10234 * (64 bytes). Max_Payload_Size is possibly modified upward in
10235 * tune_pcie_caps() which is called after this routine.
10236 */
10237}
10238
10239static void init_other(struct hfi1_devdata *dd)
10240{
10241 /* enable all CCE errors */
10242 write_csr(dd, CCE_ERR_MASK, ~0ull);
10243 /* enable *some* Misc errors */
10244 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
10245 /* enable all DC errors, except LCB */
10246 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
10247 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
10248}
10249
10250/*
10251 * Fill out the given AU table using the given CU. A CU is defined in terms
10252 * AUs. The table is a an encoding: given the index, how many AUs does that
10253 * represent?
10254 *
10255 * NOTE: Assumes that the register layout is the same for the
10256 * local and remote tables.
10257 */
10258static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
10259 u32 csr0to3, u32 csr4to7)
10260{
10261 write_csr(dd, csr0to3,
10262 0ull <<
10263 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT
10264 | 1ull <<
10265 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT
10266 | 2ull * cu <<
10267 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT
10268 | 4ull * cu <<
10269 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
10270 write_csr(dd, csr4to7,
10271 8ull * cu <<
10272 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT
10273 | 16ull * cu <<
10274 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT
10275 | 32ull * cu <<
10276 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT
10277 | 64ull * cu <<
10278 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
10279
10280}
10281
10282static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
10283{
10284 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
10285 SEND_CM_LOCAL_AU_TABLE4_TO7);
10286}
10287
10288void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
10289{
10290 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
10291 SEND_CM_REMOTE_AU_TABLE4_TO7);
10292}
10293
10294static void init_txe(struct hfi1_devdata *dd)
10295{
10296 int i;
10297
10298 /* enable all PIO, SDMA, general, and Egress errors */
10299 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
10300 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
10301 write_csr(dd, SEND_ERR_MASK, ~0ull);
10302 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
10303
10304 /* enable all per-context and per-SDMA engine errors */
10305 for (i = 0; i < dd->chip_send_contexts; i++)
10306 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
10307 for (i = 0; i < dd->chip_sdma_engines; i++)
10308 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
10309
10310 /* set the local CU to AU mapping */
10311 assign_local_cm_au_table(dd, dd->vcu);
10312
10313 /*
10314 * Set reasonable default for Credit Return Timer
10315 * Don't set on Simulator - causes it to choke.
10316 */
10317 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
10318 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
10319}
10320
10321int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
10322{
10323 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
10324 unsigned sctxt;
10325 int ret = 0;
10326 u64 reg;
10327
10328 if (!rcd || !rcd->sc) {
10329 ret = -EINVAL;
10330 goto done;
10331 }
10332 sctxt = rcd->sc->hw_context;
10333 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
10334 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
10335 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
10336 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
10337 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
10338 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
10339 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
10340 /*
10341 * Enable send-side J_KEY integrity check, unless this is A0 h/w
10342 * (due to A0 erratum).
10343 */
995deafa 10344 if (!is_ax(dd)) {
77241056
MM		 10345 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10346 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
10347 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10348 }
10349
10350 /* Enable J_KEY check on receive context. */
10351 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
10352 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
10353 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
10354 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
10355done:
10356 return ret;
10357}
10358
10359int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
10360{
10361 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
10362 unsigned sctxt;
10363 int ret = 0;
10364 u64 reg;
10365
10366 if (!rcd || !rcd->sc) {
10367 ret = -EINVAL;
10368 goto done;
10369 }
10370 sctxt = rcd->sc->hw_context;
10371 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
10372 /*
10373 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
10374 * This check would not have been enabled for A0 h/w, see
10375 * set_ctxt_jkey().
10376 */
995deafa 10377 if (!is_ax(dd)) {
77241056
MM		 10378 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10379 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
10380 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10381 }
10382 /* Turn off the J_KEY on the receive side */
10383 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
10384done:
10385 return ret;
10386}
10387
10388int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
10389{
10390 struct hfi1_ctxtdata *rcd;
10391 unsigned sctxt;
10392 int ret = 0;
10393 u64 reg;
10394
10395 if (ctxt < dd->num_rcv_contexts)
10396 rcd = dd->rcd[ctxt];
10397 else {
10398 ret = -EINVAL;
10399 goto done;
10400 }
10401 if (!rcd || !rcd->sc) {
10402 ret = -EINVAL;
10403 goto done;
10404 }
10405 sctxt = rcd->sc->hw_context;
10406 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
10407 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
10408 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
10409 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10410 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
10411 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10412done:
10413 return ret;
10414}
10415
10416int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
10417{
10418 struct hfi1_ctxtdata *rcd;
10419 unsigned sctxt;
10420 int ret = 0;
10421 u64 reg;
10422
10423 if (ctxt < dd->num_rcv_contexts)
10424 rcd = dd->rcd[ctxt];
10425 else {
10426 ret = -EINVAL;
10427 goto done;
10428 }
10429 if (!rcd || !rcd->sc) {
10430 ret = -EINVAL;
10431 goto done;
10432 }
10433 sctxt = rcd->sc->hw_context;
10434 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
10435 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
10436 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
10437 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
10438done:
10439 return ret;
10440}
10441
10442/*
10443 * Start doing the clean up the the chip. Our clean up happens in multiple
10444 * stages and this is just the first.
10445 */
10446void hfi1_start_cleanup(struct hfi1_devdata *dd)
10447{
10448 free_cntrs(dd);
10449 free_rcverr(dd);
10450 clean_up_interrupts(dd);
10451}
10452
10453#define HFI_BASE_GUID(dev) \
10454 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
10455
10456/*
10457 * Certain chip functions need to be initialized only once per asic
10458 * instead of per-device. This function finds the peer device and
10459 * checks whether that chip initialization needs to be done by this
10460 * device.
10461 */
10462static void asic_should_init(struct hfi1_devdata *dd)
10463{
10464 unsigned long flags;
10465 struct hfi1_devdata *tmp, *peer = NULL;
10466
10467 spin_lock_irqsave(&hfi1_devs_lock, flags);
10468 /* Find our peer device */
10469 list_for_each_entry(tmp, &hfi1_dev_list, list) {
10470 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
10471 dd->unit != tmp->unit) {
10472 peer = tmp;
10473 break;
10474 }
10475 }
10476
10477 /*
10478 * "Claim" the ASIC for initialization if it hasn't been
10479 " "claimed" yet.
10480 */
10481 if (!peer || !(peer->flags & HFI1_DO_INIT_ASIC))
10482 dd->flags |= HFI1_DO_INIT_ASIC;
10483 spin_unlock_irqrestore(&hfi1_devs_lock, flags);
10484}
10485
5d9157aa
DL		 10486/*
10487 * Set dd->boardname. Use a generic name if a name is not returned from
10488 * EFI variable space.
10489 *
10490 * Return 0 on success, -ENOMEM if space could not be allocated.
10491 */
10492static int obtain_boardname(struct hfi1_devdata *dd)
10493{
10494 /* generic board description */
10495 const char generic[] =
10496 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
10497 unsigned long size;
10498 int ret;
10499
10500 ret = read_hfi1_efi_var(dd, "description", &size,
10501 (void **)&dd->boardname);
10502 if (ret) {
10503 dd_dev_err(dd, "Board description not found\n");
10504 /* use generic description */
10505 dd->boardname = kstrdup(generic, GFP_KERNEL);
10506 if (!dd->boardname)
10507 return -ENOMEM;
10508 }
10509 return 0;
10510}
10511
77241056 10512/**
7c03ed85 10513 * Allocate and initialize the device structure for the hfi.
77241056
MM		 10514 * @dev: the pci_dev for hfi1_ib device
10515 * @ent: pci_device_id struct for this dev
10516 *
10517 * Also allocates, initializes, and returns the devdata struct for this
10518 * device instance
10519 *
10520 * This is global, and is called directly at init to set up the
10521 * chip-specific function pointers for later use.
10522 */
10523struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
10524 const struct pci_device_id *ent)
10525{
10526 struct hfi1_devdata *dd;
10527 struct hfi1_pportdata *ppd;
10528 u64 reg;
10529 int i, ret;
10530 static const char * const inames[] = { /* implementation names */
10531 "RTL silicon",
10532 "RTL VCS simulation",
10533 "RTL FPGA emulation",
10534 "Functional simulator"
10535 };
10536
10537 dd = hfi1_alloc_devdata(pdev,
10538 NUM_IB_PORTS * sizeof(struct hfi1_pportdata));
10539 if (IS_ERR(dd))
10540 goto bail;
10541 ppd = dd->pport;
10542 for (i = 0; i < dd->num_pports; i++, ppd++) {
10543 int vl;
10544 /* init common fields */
10545 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
10546 /* DC supports 4 link widths */
10547 ppd->link_width_supported =
10548 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
10549 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
10550 ppd->link_width_downgrade_supported =
10551 ppd->link_width_supported;
10552 /* start out enabling only 4X */
10553 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
10554 ppd->link_width_downgrade_enabled =
10555 ppd->link_width_downgrade_supported;
10556 /* link width active is 0 when link is down */
10557 /* link width downgrade active is 0 when link is down */
10558
10559 if (num_vls < HFI1_MIN_VLS_SUPPORTED
10560 || num_vls > HFI1_MAX_VLS_SUPPORTED) {
10561 hfi1_early_err(&pdev->dev,
10562 "Invalid num_vls %u, using %u VLs\n",
10563 num_vls, HFI1_MAX_VLS_SUPPORTED);
10564 num_vls = HFI1_MAX_VLS_SUPPORTED;
10565 }
10566 ppd->vls_supported = num_vls;
10567 ppd->vls_operational = ppd->vls_supported;
10568 /* Set the default MTU. */
10569 for (vl = 0; vl < num_vls; vl++)
10570 dd->vld[vl].mtu = hfi1_max_mtu;
10571 dd->vld[15].mtu = MAX_MAD_PACKET;
10572 /*
10573 * Set the initial values to reasonable default, will be set
10574 * for real when link is up.
10575 */
10576 ppd->lstate = IB_PORT_DOWN;
10577 ppd->overrun_threshold = 0x4;
10578 ppd->phy_error_threshold = 0xf;
10579 ppd->port_crc_mode_enabled = link_crc_mask;
10580 /* initialize supported LTP CRC mode */
10581 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
10582 /* initialize enabled LTP CRC mode */
10583 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
10584 /* start in offline */
10585 ppd->host_link_state = HLS_DN_OFFLINE;
10586 init_vl_arb_caches(ppd);
10587 }
10588
10589 dd->link_default = HLS_DN_POLL;
10590
10591 /*
10592 * Do remaining PCIe setup and save PCIe values in dd.
10593 * Any error printing is already done by the init code.
10594 * On return, we have the chip mapped.
10595 */
10596 ret = hfi1_pcie_ddinit(dd, pdev, ent);
10597 if (ret < 0)
10598 goto bail_free;
10599
10600 /* verify that reads actually work, save revision for reset check */
10601 dd->revision = read_csr(dd, CCE_REVISION);
10602 if (dd->revision == ~(u64)0) {
10603 dd_dev_err(dd, "cannot read chip CSRs\n");
10604 ret = -EINVAL;
10605 goto bail_cleanup;
10606 }
10607 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
10608 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
10609 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
10610 & CCE_REVISION_CHIP_REV_MINOR_MASK;
10611
10612 /* obtain the hardware ID - NOT related to unit, which is a
10613 software enumeration */
10614 reg = read_csr(dd, CCE_REVISION2);
10615 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
10616 & CCE_REVISION2_HFI_ID_MASK;
10617 /* the variable size will remove unwanted bits */
10618 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
10619 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
10620 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
10621 dd->icode < ARRAY_SIZE(inames) ? inames[dd->icode] : "unknown",
10622 (int)dd->irev);
10623
10624 /* speeds the hardware can support */
10625 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
10626 /* speeds allowed to run at */
10627 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
10628 /* give a reasonable active value, will be set on link up */
10629 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
10630
10631 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
10632 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
10633 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
10634 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
10635 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
10636 /* fix up link widths for emulation _p */
10637 ppd = dd->pport;
10638 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
10639 ppd->link_width_supported =
10640 ppd->link_width_enabled =
10641 ppd->link_width_downgrade_supported =
10642 ppd->link_width_downgrade_enabled =
10643 OPA_LINK_WIDTH_1X;
10644 }
10645 /* insure num_vls isn't larger than number of sdma engines */
10646 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
10647 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
10648 num_vls, HFI1_MAX_VLS_SUPPORTED);
10649 ppd->vls_supported = num_vls = HFI1_MAX_VLS_SUPPORTED;
10650 ppd->vls_operational = ppd->vls_supported;
10651 }
10652
10653 /*
10654 * Convert the ns parameter to the 64 * cclocks used in the CSR.
10655 * Limit the max if larger than the field holds. If timeout is
10656 * non-zero, then the calculated field will be at least 1.
10657 *
10658 * Must be after icode is set up - the cclock rate depends
10659 * on knowing the hardware being used.
10660 */
10661 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
10662 if (dd->rcv_intr_timeout_csr >
10663 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
10664 dd->rcv_intr_timeout_csr =
10665 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
10666 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
10667 dd->rcv_intr_timeout_csr = 1;
10668
7c03ed85
EH		 10669 /* needs to be done before we look for the peer device */
10670 read_guid(dd);
10671
10672 /* should this device init the ASIC block? */
10673 asic_should_init(dd);
10674
77241056
MM		 10675 /* obtain chip sizes, reset chip CSRs */
10676 init_chip(dd);
10677
10678 /* read in the PCIe link speed information */
10679 ret = pcie_speeds(dd);
10680 if (ret)
10681 goto bail_cleanup;
10682
77241056
MM		 10683 /* read in firmware */
10684 ret = hfi1_firmware_init(dd);
10685 if (ret)
10686 goto bail_cleanup;
10687
10688 /*
10689 * In general, the PCIe Gen3 transition must occur after the
10690 * chip has been idled (so it won't initiate any PCIe transactions
10691 * e.g. an interrupt) and before the driver changes any registers
10692 * (the transition will reset the registers).
10693 *
10694 * In particular, place this call after:
10695 * - init_chip() - the chip will not initiate any PCIe transactions
10696 * - pcie_speeds() - reads the current link speed
10697 * - hfi1_firmware_init() - the needed firmware is ready to be
10698 * downloaded
10699 */
10700 ret = do_pcie_gen3_transition(dd);
10701 if (ret)
10702 goto bail_cleanup;
10703
10704 /* start setting dd values and adjusting CSRs */
10705 init_early_variables(dd);
10706
10707 parse_platform_config(dd);
10708
5d9157aa
DL		 10709 ret = obtain_boardname(dd);
10710 if (ret)
77241056 10711 goto bail_cleanup;
77241056
MM		 10712
10713 snprintf(dd->boardversion, BOARD_VERS_MAX,
5d9157aa 10714 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
77241056 10715 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
77241056
MM		 10716 (u32)dd->majrev,
10717 (u32)dd->minrev,
10718 (dd->revision >> CCE_REVISION_SW_SHIFT)
10719 & CCE_REVISION_SW_MASK);
10720
10721 ret = set_up_context_variables(dd);
10722 if (ret)
10723 goto bail_cleanup;
10724
10725 /* set initial RXE CSRs */
10726 init_rxe(dd);
10727 /* set initial TXE CSRs */
10728 init_txe(dd);
10729 /* set initial non-RXE, non-TXE CSRs */
10730 init_other(dd);
10731 /* set up KDETH QP prefix in both RX and TX CSRs */
10732 init_kdeth_qp(dd);
10733
10734 /* send contexts must be set up before receive contexts */
10735 ret = init_send_contexts(dd);
10736 if (ret)
10737 goto bail_cleanup;
10738
10739 ret = hfi1_create_ctxts(dd);
10740 if (ret)
10741 goto bail_cleanup;
10742
10743 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
10744 /*
10745 * rcd[0] is guaranteed to be valid by this point. Also, all
10746 * context are using the same value, as per the module parameter.
10747 */
10748 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
10749
10750 ret = init_pervl_scs(dd);
10751 if (ret)
10752 goto bail_cleanup;
10753
10754 /* sdma init */
10755 for (i = 0; i < dd->num_pports; ++i) {
10756 ret = sdma_init(dd, i);
10757 if (ret)
10758 goto bail_cleanup;
10759 }
10760
10761 /* use contexts created by hfi1_create_ctxts */
10762 ret = set_up_interrupts(dd);
10763 if (ret)
10764 goto bail_cleanup;
10765
10766 /* set up LCB access - must be after set_up_interrupts() */
10767 init_lcb_access(dd);
10768
10769 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
10770 dd->base_guid & 0xFFFFFF);
10771
10772 dd->oui1 = dd->base_guid >> 56 & 0xFF;
10773 dd->oui2 = dd->base_guid >> 48 & 0xFF;
10774 dd->oui3 = dd->base_guid >> 40 & 0xFF;
10775
10776 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
10777 if (ret)
10778 goto bail_clear_intr;
10779 check_fabric_firmware_versions(dd);
10780
10781 thermal_init(dd);
10782
10783 ret = init_cntrs(dd);
10784 if (ret)
10785 goto bail_clear_intr;
10786
10787 ret = init_rcverr(dd);
10788 if (ret)
10789 goto bail_free_cntrs;
10790
10791 ret = eprom_init(dd);
10792 if (ret)
10793 goto bail_free_rcverr;
10794
10795 goto bail;
10796
10797bail_free_rcverr:
10798 free_rcverr(dd);
10799bail_free_cntrs:
10800 free_cntrs(dd);
10801bail_clear_intr:
10802 clean_up_interrupts(dd);
10803bail_cleanup:
10804 hfi1_pcie_ddcleanup(dd);
10805bail_free:
10806 hfi1_free_devdata(dd);
10807 dd = ERR_PTR(ret);
10808bail:
10809 return dd;
10810}
10811
10812static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
10813 u32 dw_len)
10814{
10815 u32 delta_cycles;
10816 u32 current_egress_rate = ppd->current_egress_rate;
10817 /* rates here are in units of 10^6 bits/sec */
10818
10819 if (desired_egress_rate == -1)
10820 return 0; /* shouldn't happen */
10821
10822 if (desired_egress_rate >= current_egress_rate)
10823 return 0; /* we can't help go faster, only slower */
10824
10825 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
10826 egress_cycles(dw_len * 4, current_egress_rate);
10827
10828 return (u16)delta_cycles;
10829}
10830
10831
10832/**
10833 * create_pbc - build a pbc for transmission
10834 * @flags: special case flags or-ed in built pbc
10835 * @srate: static rate
10836 * @vl: vl
10837 * @dwlen: dword length (header words + data words + pbc words)
10838 *
10839 * Create a PBC with the given flags, rate, VL, and length.
10840 *
10841 * NOTE: The PBC created will not insert any HCRC - all callers but one are
10842 * for verbs, which does not use this PSM feature. The lone other caller
10843 * is for the diagnostic interface which calls this if the user does not
10844 * supply their own PBC.
10845 */
10846u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
10847 u32 dw_len)
10848{
10849 u64 pbc, delay = 0;
10850
10851 if (unlikely(srate_mbs))
10852 delay = delay_cycles(ppd, srate_mbs, dw_len);
10853
10854 pbc = flags
10855 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
10856 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
10857 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
10858 | (dw_len & PBC_LENGTH_DWS_MASK)
10859 << PBC_LENGTH_DWS_SHIFT;
10860
10861 return pbc;
10862}
10863
10864#define SBUS_THERMAL 0x4f
10865#define SBUS_THERM_MONITOR_MODE 0x1
10866
10867#define THERM_FAILURE(dev, ret, reason) \
10868 dd_dev_err((dd), \
10869 "Thermal sensor initialization failed: %s (%d)\n", \
10870 (reason), (ret))
10871
10872/*
10873 * Initialize the Avago Thermal sensor.
10874 *
10875 * After initialization, enable polling of thermal sensor through
10876 * SBus interface. In order for this to work, the SBus Master
10877 * firmware has to be loaded due to the fact that the HW polling
10878 * logic uses SBus interrupts, which are not supported with
10879 * default firmware. Otherwise, no data will be returned through
10880 * the ASIC_STS_THERM CSR.
10881 */
10882static int thermal_init(struct hfi1_devdata *dd)
10883{
10884 int ret = 0;
10885
10886 if (dd->icode != ICODE_RTL_SILICON ||
10887 !(dd->flags & HFI1_DO_INIT_ASIC))
10888 return ret;
10889
10890 acquire_hw_mutex(dd);
10891 dd_dev_info(dd, "Initializing thermal sensor\n");
4ef98989
JAQ		 10892 /* Disable polling of thermal readings */
10893 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
10894 msleep(100);
77241056
MM		 10895 /* Thermal Sensor Initialization */
10896 /* Step 1: Reset the Thermal SBus Receiver */
10897 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10898 RESET_SBUS_RECEIVER, 0);
10899 if (ret) {
10900 THERM_FAILURE(dd, ret, "Bus Reset");
10901 goto done;
10902 }
10903 /* Step 2: Set Reset bit in Thermal block */
10904 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10905 WRITE_SBUS_RECEIVER, 0x1);
10906 if (ret) {
10907 THERM_FAILURE(dd, ret, "Therm Block Reset");
10908 goto done;
10909 }
10910 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
10911 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
10912 WRITE_SBUS_RECEIVER, 0x32);
10913 if (ret) {
10914 THERM_FAILURE(dd, ret, "Write Clock Div");
10915 goto done;
10916 }
10917 /* Step 4: Select temperature mode */
10918 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
10919 WRITE_SBUS_RECEIVER,
10920 SBUS_THERM_MONITOR_MODE);
10921 if (ret) {
10922 THERM_FAILURE(dd, ret, "Write Mode Sel");
10923 goto done;
10924 }
10925 /* Step 5: De-assert block reset and start conversion */
10926 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
10927 WRITE_SBUS_RECEIVER, 0x2);
10928 if (ret) {
10929 THERM_FAILURE(dd, ret, "Write Reset Deassert");
10930 goto done;
10931 }
10932 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
10933 msleep(22);
10934
10935 /* Enable polling of thermal readings */
10936 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
10937done:
10938 release_hw_mutex(dd);
10939 return ret;
10940}
10941
10942static void handle_temp_err(struct hfi1_devdata *dd)
10943{
10944 struct hfi1_pportdata *ppd = &dd->pport[0];
10945 /*
10946 * Thermal Critical Interrupt
10947 * Put the device into forced freeze mode, take link down to
10948 * offline, and put DC into reset.
10949 */
10950 dd_dev_emerg(dd,
10951 "Critical temperature reached! Forcing device into freeze mode!\n");
10952 dd->flags |= HFI1_FORCED_FREEZE;
10953 start_freeze_handling(ppd, FREEZE_SELF|FREEZE_ABORT);
10954 /*
10955 * Shut DC down as much and as quickly as possible.
10956 *
10957 * Step 1: Take the link down to OFFLINE. This will cause the
10958 * 8051 to put the Serdes in reset. However, we don't want to
10959 * go through the entire link state machine since we want to
10960 * shutdown ASAP. Furthermore, this is not a graceful shutdown
10961 * but rather an attempt to save the chip.
10962 * Code below is almost the same as quiet_serdes() but avoids
10963 * all the extra work and the sleeps.
10964 */
10965 ppd->driver_link_ready = 0;
10966 ppd->link_enabled = 0;
10967 set_physical_link_state(dd, PLS_OFFLINE |
10968 (OPA_LINKDOWN_REASON_SMA_DISABLED << 8));
10969 /*
10970 * Step 2: Shutdown LCB and 8051
10971 * After shutdown, do not restore DC_CFG_RESET value.
10972 */
10973 dc_shutdown(dd);
10974}
Linux kernel - Deliverables
This page took 0.514614 seconds and 5 git commands to generate.