2 * Copyright (c) 2016 Hisilicon Limited.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/platform_device.h>
34 #include <linux/acpi.h>
35 #include <rdma/ib_umem.h>
36 #include "hns_roce_common.h"
37 #include "hns_roce_device.h"
38 #include "hns_roce_cmd.h"
39 #include "hns_roce_hem.h"
40 #include "hns_roce_hw_v1.h"
42 static void set_data_seg(struct hns_roce_wqe_data_seg
*dseg
, struct ib_sge
*sg
)
44 dseg
->lkey
= cpu_to_le32(sg
->lkey
);
45 dseg
->addr
= cpu_to_le64(sg
->addr
);
46 dseg
->len
= cpu_to_le32(sg
->length
);
49 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg
*rseg
, u64 remote_addr
,
52 rseg
->raddr
= cpu_to_le64(remote_addr
);
53 rseg
->rkey
= cpu_to_le32(rkey
);
57 int hns_roce_v1_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
58 struct ib_send_wr
**bad_wr
)
60 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
61 struct hns_roce_ah
*ah
= to_hr_ah(ud_wr(wr
)->ah
);
62 struct hns_roce_ud_send_wqe
*ud_sq_wqe
= NULL
;
63 struct hns_roce_wqe_ctrl_seg
*ctrl
= NULL
;
64 struct hns_roce_wqe_data_seg
*dseg
= NULL
;
65 struct hns_roce_qp
*qp
= to_hr_qp(ibqp
);
66 struct device
*dev
= &hr_dev
->pdev
->dev
;
67 struct hns_roce_sq_db sq_db
;
68 int ps_opcode
= 0, i
= 0;
69 unsigned long flags
= 0;
76 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
78 ind
= qp
->sq_next_wqe
;
79 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
80 if (hns_roce_wq_overflow(&qp
->sq
, nreq
, qp
->ibqp
.send_cq
)) {
86 if (unlikely(wr
->num_sge
> qp
->sq
.max_gs
)) {
87 dev_err(dev
, "num_sge=%d > qp->sq.max_gs=%d\n",
88 wr
->num_sge
, qp
->sq
.max_gs
);
94 wqe
= get_send_wqe(qp
, ind
& (qp
->sq
.wqe_cnt
- 1));
95 qp
->sq
.wrid
[(qp
->sq
.head
+ nreq
) & (qp
->sq
.wqe_cnt
- 1)] =
98 /* Corresponding to the RC and RD type wqe process separately */
99 if (ibqp
->qp_type
== IB_QPT_GSI
) {
101 roce_set_field(ud_sq_wqe
->dmac_h
,
102 UD_SEND_WQE_U32_4_DMAC_0_M
,
103 UD_SEND_WQE_U32_4_DMAC_0_S
,
105 roce_set_field(ud_sq_wqe
->dmac_h
,
106 UD_SEND_WQE_U32_4_DMAC_1_M
,
107 UD_SEND_WQE_U32_4_DMAC_1_S
,
109 roce_set_field(ud_sq_wqe
->dmac_h
,
110 UD_SEND_WQE_U32_4_DMAC_2_M
,
111 UD_SEND_WQE_U32_4_DMAC_2_S
,
113 roce_set_field(ud_sq_wqe
->dmac_h
,
114 UD_SEND_WQE_U32_4_DMAC_3_M
,
115 UD_SEND_WQE_U32_4_DMAC_3_S
,
118 roce_set_field(ud_sq_wqe
->u32_8
,
119 UD_SEND_WQE_U32_8_DMAC_4_M
,
120 UD_SEND_WQE_U32_8_DMAC_4_S
,
122 roce_set_field(ud_sq_wqe
->u32_8
,
123 UD_SEND_WQE_U32_8_DMAC_5_M
,
124 UD_SEND_WQE_U32_8_DMAC_5_S
,
126 roce_set_field(ud_sq_wqe
->u32_8
,
127 UD_SEND_WQE_U32_8_OPERATION_TYPE_M
,
128 UD_SEND_WQE_U32_8_OPERATION_TYPE_S
,
129 HNS_ROCE_WQE_OPCODE_SEND
);
130 roce_set_field(ud_sq_wqe
->u32_8
,
131 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M
,
132 UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S
,
134 roce_set_bit(ud_sq_wqe
->u32_8
,
135 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S
,
138 ud_sq_wqe
->u32_8
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
139 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
140 (wr
->send_flags
& IB_SEND_SOLICITED
?
141 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
142 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
) ?
143 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0);
145 roce_set_field(ud_sq_wqe
->u32_16
,
146 UD_SEND_WQE_U32_16_DEST_QP_M
,
147 UD_SEND_WQE_U32_16_DEST_QP_S
,
148 ud_wr(wr
)->remote_qpn
);
149 roce_set_field(ud_sq_wqe
->u32_16
,
150 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M
,
151 UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S
,
154 roce_set_field(ud_sq_wqe
->u32_36
,
155 UD_SEND_WQE_U32_36_FLOW_LABEL_M
,
156 UD_SEND_WQE_U32_36_FLOW_LABEL_S
, 0);
157 roce_set_field(ud_sq_wqe
->u32_36
,
158 UD_SEND_WQE_U32_36_PRIORITY_M
,
159 UD_SEND_WQE_U32_36_PRIORITY_S
,
160 ah
->av
.sl_tclass_flowlabel
>>
162 roce_set_field(ud_sq_wqe
->u32_36
,
163 UD_SEND_WQE_U32_36_SGID_INDEX_M
,
164 UD_SEND_WQE_U32_36_SGID_INDEX_S
,
165 hns_get_gid_index(hr_dev
, qp
->port
,
168 roce_set_field(ud_sq_wqe
->u32_40
,
169 UD_SEND_WQE_U32_40_HOP_LIMIT_M
,
170 UD_SEND_WQE_U32_40_HOP_LIMIT_S
,
172 roce_set_field(ud_sq_wqe
->u32_40
,
173 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M
,
174 UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S
, 0);
176 memcpy(&ud_sq_wqe
->dgid
[0], &ah
->av
.dgid
[0], GID_LEN
);
178 ud_sq_wqe
->va0_l
= (u32
)wr
->sg_list
[0].addr
;
179 ud_sq_wqe
->va0_h
= (wr
->sg_list
[0].addr
) >> 32;
180 ud_sq_wqe
->l_key0
= wr
->sg_list
[0].lkey
;
182 ud_sq_wqe
->va1_l
= (u32
)wr
->sg_list
[1].addr
;
183 ud_sq_wqe
->va1_h
= (wr
->sg_list
[1].addr
) >> 32;
184 ud_sq_wqe
->l_key1
= wr
->sg_list
[1].lkey
;
186 } else if (ibqp
->qp_type
== IB_QPT_RC
) {
188 memset(ctrl
, 0, sizeof(struct hns_roce_wqe_ctrl_seg
));
189 for (i
= 0; i
< wr
->num_sge
; i
++)
190 ctrl
->msg_length
+= wr
->sg_list
[i
].length
;
194 ctrl
->imm_data
= send_ieth(wr
);
196 /*Ctrl field, ctrl set type: sig, solic, imm, fence */
197 /* SO wait for conforming application scenarios */
198 ctrl
->flag
|= (wr
->send_flags
& IB_SEND_SIGNALED
?
199 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY
) : 0) |
200 (wr
->send_flags
& IB_SEND_SOLICITED
?
201 cpu_to_le32(HNS_ROCE_WQE_SE
) : 0) |
202 ((wr
->opcode
== IB_WR_SEND_WITH_IMM
||
203 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) ?
204 cpu_to_le32(HNS_ROCE_WQE_IMM
) : 0) |
205 (wr
->send_flags
& IB_SEND_FENCE
?
206 (cpu_to_le32(HNS_ROCE_WQE_FENCE
)) : 0);
208 wqe
= (struct hns_roce_wqe_ctrl_seg
*)wqe
+
209 sizeof(struct hns_roce_wqe_ctrl_seg
);
211 switch (wr
->opcode
) {
212 case IB_WR_RDMA_READ
:
213 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_READ
;
214 set_raddr_seg(wqe
, atomic_wr(wr
)->remote_addr
,
215 atomic_wr(wr
)->rkey
);
217 case IB_WR_RDMA_WRITE
:
218 case IB_WR_RDMA_WRITE_WITH_IMM
:
219 ps_opcode
= HNS_ROCE_WQE_OPCODE_RDMA_WRITE
;
220 set_raddr_seg(wqe
, atomic_wr(wr
)->remote_addr
,
221 atomic_wr(wr
)->rkey
);
224 case IB_WR_SEND_WITH_INV
:
225 case IB_WR_SEND_WITH_IMM
:
226 ps_opcode
= HNS_ROCE_WQE_OPCODE_SEND
;
228 case IB_WR_LOCAL_INV
:
230 case IB_WR_ATOMIC_CMP_AND_SWP
:
231 case IB_WR_ATOMIC_FETCH_AND_ADD
:
234 ps_opcode
= HNS_ROCE_WQE_OPCODE_MASK
;
237 ctrl
->flag
|= cpu_to_le32(ps_opcode
);
238 wqe
= (struct hns_roce_wqe_raddr_seg
*)wqe
+
239 sizeof(struct hns_roce_wqe_raddr_seg
);
242 if (wr
->send_flags
& IB_SEND_INLINE
&& wr
->num_sge
) {
243 if (ctrl
->msg_length
>
244 hr_dev
->caps
.max_sq_inline
) {
247 dev_err(dev
, "inline len(1-%d)=%d, illegal",
249 hr_dev
->caps
.max_sq_inline
);
252 for (i
= 0; i
< wr
->num_sge
; i
++) {
253 memcpy(wqe
, ((void *) (uintptr_t)
254 wr
->sg_list
[i
].addr
),
255 wr
->sg_list
[i
].length
);
256 wqe
= (struct hns_roce_wqe_raddr_seg
*)
257 wqe
+ wr
->sg_list
[i
].length
;
259 ctrl
->flag
|= HNS_ROCE_WQE_INLINE
;
262 for (i
= 0; i
< wr
->num_sge
; i
++)
263 set_data_seg(dseg
+ i
, wr
->sg_list
+ i
);
265 ctrl
->flag
|= cpu_to_le32(wr
->num_sge
<<
266 HNS_ROCE_WQE_SGE_NUM_BIT
);
270 dev_dbg(dev
, "unSupported QP type\n");
284 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_SQ_HEAD_M
,
285 SQ_DOORBELL_U32_4_SQ_HEAD_S
,
286 (qp
->sq
.head
& ((qp
->sq
.wqe_cnt
<< 1) - 1)));
287 roce_set_field(sq_db
.u32_4
, SQ_DOORBELL_U32_4_PORT_M
,
288 SQ_DOORBELL_U32_4_PORT_S
, qp
->port
);
289 roce_set_field(sq_db
.u32_8
, SQ_DOORBELL_U32_8_QPN_M
,
290 SQ_DOORBELL_U32_8_QPN_S
, qp
->doorbell_qpn
);
291 roce_set_bit(sq_db
.u32_8
, SQ_DOORBELL_HW_SYNC_S
, 1);
293 doorbell
[0] = sq_db
.u32_4
;
294 doorbell
[1] = sq_db
.u32_8
;
296 hns_roce_write64_k(doorbell
, qp
->sq
.db_reg_l
);
297 qp
->sq_next_wqe
= ind
;
300 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
305 int hns_roce_v1_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
306 struct ib_recv_wr
**bad_wr
)
313 unsigned long flags
= 0;
314 struct hns_roce_rq_wqe_ctrl
*ctrl
= NULL
;
315 struct hns_roce_wqe_data_seg
*scat
= NULL
;
316 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
317 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
318 struct device
*dev
= &hr_dev
->pdev
->dev
;
319 struct hns_roce_rq_db rq_db
;
320 uint32_t doorbell
[2] = {0};
322 spin_lock_irqsave(&hr_qp
->rq
.lock
, flags
);
323 ind
= hr_qp
->rq
.head
& (hr_qp
->rq
.wqe_cnt
- 1);
325 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
326 if (hns_roce_wq_overflow(&hr_qp
->rq
, nreq
,
327 hr_qp
->ibqp
.recv_cq
)) {
333 if (unlikely(wr
->num_sge
> hr_qp
->rq
.max_gs
)) {
334 dev_err(dev
, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
335 wr
->num_sge
, hr_qp
->rq
.max_gs
);
341 ctrl
= get_recv_wqe(hr_qp
, ind
);
343 roce_set_field(ctrl
->rwqe_byte_12
,
344 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M
,
345 RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S
,
348 scat
= (struct hns_roce_wqe_data_seg
*)(ctrl
+ 1);
350 for (i
= 0; i
< wr
->num_sge
; i
++)
351 set_data_seg(scat
+ i
, wr
->sg_list
+ i
);
353 hr_qp
->rq
.wrid
[ind
] = wr
->wr_id
;
355 ind
= (ind
+ 1) & (hr_qp
->rq
.wqe_cnt
- 1);
360 hr_qp
->rq
.head
+= nreq
;
364 if (ibqp
->qp_type
== IB_QPT_GSI
) {
365 /* SW update GSI rq header */
366 reg_val
= roce_read(to_hr_dev(ibqp
->device
),
367 ROCEE_QP1C_CFG3_0_REG
+
368 QP1C_CFGN_OFFSET
* hr_qp
->port
);
369 roce_set_field(reg_val
,
370 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M
,
371 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S
,
373 roce_write(to_hr_dev(ibqp
->device
),
374 ROCEE_QP1C_CFG3_0_REG
+
375 QP1C_CFGN_OFFSET
* hr_qp
->port
, reg_val
);
380 roce_set_field(rq_db
.u32_4
, RQ_DOORBELL_U32_4_RQ_HEAD_M
,
381 RQ_DOORBELL_U32_4_RQ_HEAD_S
,
383 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_QPN_M
,
384 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
385 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_CMD_M
,
386 RQ_DOORBELL_U32_8_CMD_S
, 1);
387 roce_set_bit(rq_db
.u32_8
, RQ_DOORBELL_U32_8_HW_SYNC_S
,
390 doorbell
[0] = rq_db
.u32_4
;
391 doorbell
[1] = rq_db
.u32_8
;
393 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
396 spin_unlock_irqrestore(&hr_qp
->rq
.lock
, flags
);
401 static void hns_roce_set_db_event_mode(struct hns_roce_dev
*hr_dev
,
402 int sdb_mode
, int odb_mode
)
406 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
407 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S
, sdb_mode
);
408 roce_set_bit(val
, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S
, odb_mode
);
409 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
412 static void hns_roce_set_db_ext_mode(struct hns_roce_dev
*hr_dev
, u32 sdb_mode
,
417 /* Configure SDB/ODB extend mode */
418 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
419 roce_set_bit(val
, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S
, sdb_mode
);
420 roce_set_bit(val
, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S
, odb_mode
);
421 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
424 static void hns_roce_set_sdb(struct hns_roce_dev
*hr_dev
, u32 sdb_alept
,
430 val
= roce_read(hr_dev
, ROCEE_DB_SQ_WL_REG
);
431 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M
,
432 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S
, sdb_alful
);
433 roce_set_field(val
, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M
,
434 ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S
, sdb_alept
);
435 roce_write(hr_dev
, ROCEE_DB_SQ_WL_REG
, val
);
438 static void hns_roce_set_odb(struct hns_roce_dev
*hr_dev
, u32 odb_alept
,
444 val
= roce_read(hr_dev
, ROCEE_DB_OTHERS_WL_REG
);
445 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M
,
446 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S
, odb_alful
);
447 roce_set_field(val
, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M
,
448 ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S
, odb_alept
);
449 roce_write(hr_dev
, ROCEE_DB_OTHERS_WL_REG
, val
);
452 static void hns_roce_set_sdb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_sdb_alept
,
455 struct device
*dev
= &hr_dev
->pdev
->dev
;
456 struct hns_roce_v1_priv
*priv
;
457 struct hns_roce_db_table
*db
;
458 dma_addr_t sdb_dma_addr
;
461 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
462 db
= &priv
->db_table
;
464 /* Configure extend SDB threshold */
465 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_EMPTY_REG
, ext_sdb_alept
);
466 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_WL_REG
, ext_sdb_alful
);
468 /* Configure extend SDB base addr */
469 sdb_dma_addr
= db
->ext_db
->sdb_buf_list
->map
;
470 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_REG
, (u32
)(sdb_dma_addr
>> 12));
472 /* Configure extend SDB depth */
473 val
= roce_read(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
);
474 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M
,
475 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S
,
476 db
->ext_db
->esdb_dep
);
478 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
479 * using 4K page, and shift more 32 because of
480 * caculating the high 32 bit value evaluated to hardware.
482 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M
,
483 ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S
, sdb_dma_addr
>> 44);
484 roce_write(hr_dev
, ROCEE_EXT_DB_SQ_H_REG
, val
);
486 dev_dbg(dev
, "ext SDB depth: 0x%x\n", db
->ext_db
->esdb_dep
);
487 dev_dbg(dev
, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
488 ext_sdb_alept
, ext_sdb_alful
);
491 static void hns_roce_set_odb_ext(struct hns_roce_dev
*hr_dev
, u32 ext_odb_alept
,
494 struct device
*dev
= &hr_dev
->pdev
->dev
;
495 struct hns_roce_v1_priv
*priv
;
496 struct hns_roce_db_table
*db
;
497 dma_addr_t odb_dma_addr
;
500 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
501 db
= &priv
->db_table
;
503 /* Configure extend ODB threshold */
504 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG
, ext_odb_alept
);
505 roce_write(hr_dev
, ROCEE_EXT_DB_OTHERS_WL_REG
, ext_odb_alful
);
507 /* Configure extend ODB base addr */
508 odb_dma_addr
= db
->ext_db
->odb_buf_list
->map
;
509 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_REG
, (u32
)(odb_dma_addr
>> 12));
511 /* Configure extend ODB depth */
512 val
= roce_read(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
);
513 roce_set_field(val
, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M
,
514 ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S
,
515 db
->ext_db
->eodb_dep
);
516 roce_set_field(val
, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M
,
517 ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S
,
518 db
->ext_db
->eodb_dep
);
519 roce_write(hr_dev
, ROCEE_EXT_DB_OTH_H_REG
, val
);
521 dev_dbg(dev
, "ext ODB depth: 0x%x\n", db
->ext_db
->eodb_dep
);
522 dev_dbg(dev
, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
523 ext_odb_alept
, ext_odb_alful
);
526 static int hns_roce_db_ext_init(struct hns_roce_dev
*hr_dev
, u32 sdb_ext_mod
,
529 struct device
*dev
= &hr_dev
->pdev
->dev
;
530 struct hns_roce_v1_priv
*priv
;
531 struct hns_roce_db_table
*db
;
532 dma_addr_t sdb_dma_addr
;
533 dma_addr_t odb_dma_addr
;
536 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
537 db
= &priv
->db_table
;
539 db
->ext_db
= kmalloc(sizeof(*db
->ext_db
), GFP_KERNEL
);
544 db
->ext_db
->sdb_buf_list
= kmalloc(
545 sizeof(*db
->ext_db
->sdb_buf_list
), GFP_KERNEL
);
546 if (!db
->ext_db
->sdb_buf_list
) {
548 goto ext_sdb_buf_fail_out
;
551 db
->ext_db
->sdb_buf_list
->buf
= dma_alloc_coherent(dev
,
552 HNS_ROCE_V1_EXT_SDB_SIZE
,
553 &sdb_dma_addr
, GFP_KERNEL
);
554 if (!db
->ext_db
->sdb_buf_list
->buf
) {
556 goto alloc_sq_db_buf_fail
;
558 db
->ext_db
->sdb_buf_list
->map
= sdb_dma_addr
;
560 db
->ext_db
->esdb_dep
= ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH
);
561 hns_roce_set_sdb_ext(hr_dev
, HNS_ROCE_V1_EXT_SDB_ALEPT
,
562 HNS_ROCE_V1_EXT_SDB_ALFUL
);
564 hns_roce_set_sdb(hr_dev
, HNS_ROCE_V1_SDB_ALEPT
,
565 HNS_ROCE_V1_SDB_ALFUL
);
568 db
->ext_db
->odb_buf_list
= kmalloc(
569 sizeof(*db
->ext_db
->odb_buf_list
), GFP_KERNEL
);
570 if (!db
->ext_db
->odb_buf_list
) {
572 goto ext_odb_buf_fail_out
;
575 db
->ext_db
->odb_buf_list
->buf
= dma_alloc_coherent(dev
,
576 HNS_ROCE_V1_EXT_ODB_SIZE
,
577 &odb_dma_addr
, GFP_KERNEL
);
578 if (!db
->ext_db
->odb_buf_list
->buf
) {
580 goto alloc_otr_db_buf_fail
;
582 db
->ext_db
->odb_buf_list
->map
= odb_dma_addr
;
584 db
->ext_db
->eodb_dep
= ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH
);
585 hns_roce_set_odb_ext(hr_dev
, HNS_ROCE_V1_EXT_ODB_ALEPT
,
586 HNS_ROCE_V1_EXT_ODB_ALFUL
);
588 hns_roce_set_odb(hr_dev
, HNS_ROCE_V1_ODB_ALEPT
,
589 HNS_ROCE_V1_ODB_ALFUL
);
591 hns_roce_set_db_ext_mode(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
595 alloc_otr_db_buf_fail
:
596 kfree(db
->ext_db
->odb_buf_list
);
598 ext_odb_buf_fail_out
:
600 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
601 db
->ext_db
->sdb_buf_list
->buf
,
602 db
->ext_db
->sdb_buf_list
->map
);
605 alloc_sq_db_buf_fail
:
607 kfree(db
->ext_db
->sdb_buf_list
);
609 ext_sdb_buf_fail_out
:
614 static int hns_roce_db_init(struct hns_roce_dev
*hr_dev
)
616 struct device
*dev
= &hr_dev
->pdev
->dev
;
617 struct hns_roce_v1_priv
*priv
;
618 struct hns_roce_db_table
*db
;
625 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
626 db
= &priv
->db_table
;
628 memset(db
, 0, sizeof(*db
));
630 /* Default DB mode */
631 sdb_ext_mod
= HNS_ROCE_SDB_EXTEND_MODE
;
632 odb_ext_mod
= HNS_ROCE_ODB_EXTEND_MODE
;
633 sdb_evt_mod
= HNS_ROCE_SDB_NORMAL_MODE
;
634 odb_evt_mod
= HNS_ROCE_ODB_POLL_MODE
;
636 db
->sdb_ext_mod
= sdb_ext_mod
;
637 db
->odb_ext_mod
= odb_ext_mod
;
640 ret
= hns_roce_db_ext_init(hr_dev
, sdb_ext_mod
, odb_ext_mod
);
642 dev_err(dev
, "Failed in extend DB configuration.\n");
646 hns_roce_set_db_event_mode(hr_dev
, sdb_evt_mod
, odb_evt_mod
);
651 static void hns_roce_db_free(struct hns_roce_dev
*hr_dev
)
653 struct device
*dev
= &hr_dev
->pdev
->dev
;
654 struct hns_roce_v1_priv
*priv
;
655 struct hns_roce_db_table
*db
;
657 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
658 db
= &priv
->db_table
;
660 if (db
->sdb_ext_mod
) {
661 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_SDB_SIZE
,
662 db
->ext_db
->sdb_buf_list
->buf
,
663 db
->ext_db
->sdb_buf_list
->map
);
664 kfree(db
->ext_db
->sdb_buf_list
);
667 if (db
->odb_ext_mod
) {
668 dma_free_coherent(dev
, HNS_ROCE_V1_EXT_ODB_SIZE
,
669 db
->ext_db
->odb_buf_list
->buf
,
670 db
->ext_db
->odb_buf_list
->map
);
671 kfree(db
->ext_db
->odb_buf_list
);
677 static int hns_roce_raq_init(struct hns_roce_dev
*hr_dev
)
683 struct hns_roce_v1_priv
*priv
;
684 struct hns_roce_raq_table
*raq
;
685 struct device
*dev
= &hr_dev
->pdev
->dev
;
687 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
688 raq
= &priv
->raq_table
;
690 raq
->e_raq_buf
= kzalloc(sizeof(*(raq
->e_raq_buf
)), GFP_KERNEL
);
694 raq
->e_raq_buf
->buf
= dma_alloc_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
,
696 if (!raq
->e_raq_buf
->buf
) {
698 goto err_dma_alloc_raq
;
700 raq
->e_raq_buf
->map
= addr
;
702 /* Configure raq extended address. 48bit 4K align*/
703 roce_write(hr_dev
, ROCEE_EXT_RAQ_REG
, raq
->e_raq_buf
->map
>> 12);
705 /* Configure raq_shift */
706 raq_shift
= ilog2(HNS_ROCE_V1_RAQ_SIZE
/ HNS_ROCE_V1_RAQ_ENTRY
);
707 val
= roce_read(hr_dev
, ROCEE_EXT_RAQ_H_REG
);
708 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M
,
709 ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S
, raq_shift
);
711 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
712 * using 4K page, and shift more 32 because of
713 * caculating the high 32 bit value evaluated to hardware.
715 roce_set_field(val
, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M
,
716 ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S
,
717 raq
->e_raq_buf
->map
>> 44);
718 roce_write(hr_dev
, ROCEE_EXT_RAQ_H_REG
, val
);
719 dev_dbg(dev
, "Configure raq_shift 0x%x.\n", val
);
721 /* Configure raq threshold */
722 val
= roce_read(hr_dev
, ROCEE_RAQ_WL_REG
);
723 roce_set_field(val
, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M
,
724 ROCEE_RAQ_WL_ROCEE_RAQ_WL_S
,
725 HNS_ROCE_V1_EXT_RAQ_WF
);
726 roce_write(hr_dev
, ROCEE_RAQ_WL_REG
, val
);
727 dev_dbg(dev
, "Configure raq_wl 0x%x.\n", val
);
729 /* Enable extend raq */
730 val
= roce_read(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
);
732 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M
,
733 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S
,
734 POL_TIME_INTERVAL_VAL
);
735 roce_set_bit(val
, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE
, 1);
737 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M
,
738 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S
,
741 ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S
, 1);
742 roce_write(hr_dev
, ROCEE_WRMS_POL_TIME_INTERVAL_REG
, val
);
743 dev_dbg(dev
, "Configure WrmsPolTimeInterval 0x%x.\n", val
);
745 /* Enable raq drop */
746 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
747 roce_set_bit(val
, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S
, 1);
748 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
749 dev_dbg(dev
, "Configure GlbCfg = 0x%x.\n", val
);
754 kfree(raq
->e_raq_buf
);
758 static void hns_roce_raq_free(struct hns_roce_dev
*hr_dev
)
760 struct device
*dev
= &hr_dev
->pdev
->dev
;
761 struct hns_roce_v1_priv
*priv
;
762 struct hns_roce_raq_table
*raq
;
764 priv
= (struct hns_roce_v1_priv
*)hr_dev
->hw
->priv
;
765 raq
= &priv
->raq_table
;
767 dma_free_coherent(dev
, HNS_ROCE_V1_RAQ_SIZE
, raq
->e_raq_buf
->buf
,
768 raq
->e_raq_buf
->map
);
769 kfree(raq
->e_raq_buf
);
772 static void hns_roce_port_enable(struct hns_roce_dev
*hr_dev
, int enable_flag
)
777 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
779 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
780 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
,
782 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
784 val
= roce_read(hr_dev
, ROCEE_GLB_CFG_REG
);
785 /* Close all ports */
786 roce_set_field(val
, ROCEE_GLB_CFG_ROCEE_PORT_ST_M
,
787 ROCEE_GLB_CFG_ROCEE_PORT_ST_S
, 0x0);
788 roce_write(hr_dev
, ROCEE_GLB_CFG_REG
, val
);
793 * hns_roce_v1_reset - reset RoCE
794 * @hr_dev: RoCE device struct pointer
795 * @enable: true -- drop reset, false -- reset
796 * return 0 - success , negative --fail
798 int hns_roce_v1_reset(struct hns_roce_dev
*hr_dev
, bool dereset
)
800 struct device_node
*dsaf_node
;
801 struct device
*dev
= &hr_dev
->pdev
->dev
;
802 struct device_node
*np
= dev
->of_node
;
803 struct fwnode_handle
*fwnode
;
806 /* check if this is DT/ACPI case */
807 if (dev_of_node(dev
)) {
808 dsaf_node
= of_parse_phandle(np
, "dsaf-handle", 0);
810 dev_err(dev
, "could not find dsaf-handle\n");
813 fwnode
= &dsaf_node
->fwnode
;
814 } else if (is_acpi_device_node(dev
->fwnode
)) {
815 struct acpi_reference_args args
;
817 ret
= acpi_node_get_property_reference(dev
->fwnode
,
818 "dsaf-handle", 0, &args
);
820 dev_err(dev
, "could not find dsaf-handle\n");
823 fwnode
= acpi_fwnode_handle(args
.adev
);
825 dev_err(dev
, "cannot read data from DT or ACPI\n");
829 ret
= hns_dsaf_roce_reset(fwnode
, false);
834 msleep(SLEEP_TIME_INTERVAL
);
835 ret
= hns_dsaf_roce_reset(fwnode
, true);
841 void hns_roce_v1_profile(struct hns_roce_dev
*hr_dev
)
844 struct hns_roce_caps
*caps
= &hr_dev
->caps
;
846 hr_dev
->vendor_id
= le32_to_cpu(roce_read(hr_dev
, ROCEE_VENDOR_ID_REG
));
847 hr_dev
->vendor_part_id
= le32_to_cpu(roce_read(hr_dev
,
848 ROCEE_VENDOR_PART_ID_REG
));
849 hr_dev
->hw_rev
= le32_to_cpu(roce_read(hr_dev
, ROCEE_HW_VERSION_REG
));
851 hr_dev
->sys_image_guid
= le32_to_cpu(roce_read(hr_dev
,
852 ROCEE_SYS_IMAGE_GUID_L_REG
)) |
853 ((u64
)le32_to_cpu(roce_read(hr_dev
,
854 ROCEE_SYS_IMAGE_GUID_H_REG
)) << 32);
856 caps
->num_qps
= HNS_ROCE_V1_MAX_QP_NUM
;
857 caps
->max_wqes
= HNS_ROCE_V1_MAX_WQE_NUM
;
858 caps
->num_cqs
= HNS_ROCE_V1_MAX_CQ_NUM
;
859 caps
->max_cqes
= HNS_ROCE_V1_MAX_CQE_NUM
;
860 caps
->max_sq_sg
= HNS_ROCE_V1_SG_NUM
;
861 caps
->max_rq_sg
= HNS_ROCE_V1_SG_NUM
;
862 caps
->max_sq_inline
= HNS_ROCE_V1_INLINE_SIZE
;
863 caps
->num_uars
= HNS_ROCE_V1_UAR_NUM
;
864 caps
->phy_num_uars
= HNS_ROCE_V1_PHY_UAR_NUM
;
865 caps
->num_aeq_vectors
= HNS_ROCE_AEQE_VEC_NUM
;
866 caps
->num_comp_vectors
= HNS_ROCE_COMP_VEC_NUM
;
867 caps
->num_other_vectors
= HNS_ROCE_AEQE_OF_VEC_NUM
;
868 caps
->num_mtpts
= HNS_ROCE_V1_MAX_MTPT_NUM
;
869 caps
->num_mtt_segs
= HNS_ROCE_V1_MAX_MTT_SEGS
;
870 caps
->num_pds
= HNS_ROCE_V1_MAX_PD_NUM
;
871 caps
->max_qp_init_rdma
= HNS_ROCE_V1_MAX_QP_INIT_RDMA
;
872 caps
->max_qp_dest_rdma
= HNS_ROCE_V1_MAX_QP_DEST_RDMA
;
873 caps
->max_sq_desc_sz
= HNS_ROCE_V1_MAX_SQ_DESC_SZ
;
874 caps
->max_rq_desc_sz
= HNS_ROCE_V1_MAX_RQ_DESC_SZ
;
875 caps
->qpc_entry_sz
= HNS_ROCE_V1_QPC_ENTRY_SIZE
;
876 caps
->irrl_entry_sz
= HNS_ROCE_V1_IRRL_ENTRY_SIZE
;
877 caps
->cqc_entry_sz
= HNS_ROCE_V1_CQC_ENTRY_SIZE
;
878 caps
->mtpt_entry_sz
= HNS_ROCE_V1_MTPT_ENTRY_SIZE
;
879 caps
->mtt_entry_sz
= HNS_ROCE_V1_MTT_ENTRY_SIZE
;
880 caps
->cq_entry_sz
= HNS_ROCE_V1_CQE_ENTRY_SIZE
;
881 caps
->page_size_cap
= HNS_ROCE_V1_PAGE_SIZE_SUPPORT
;
883 caps
->reserved_lkey
= 0;
884 caps
->reserved_pds
= 0;
885 caps
->reserved_mrws
= 1;
886 caps
->reserved_uars
= 0;
887 caps
->reserved_cqs
= 0;
889 for (i
= 0; i
< caps
->num_ports
; i
++)
890 caps
->pkey_table_len
[i
] = 1;
892 for (i
= 0; i
< caps
->num_ports
; i
++) {
893 /* Six ports shared 16 GID in v1 engine */
894 if (i
>= (HNS_ROCE_V1_GID_NUM
% caps
->num_ports
))
895 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
898 caps
->gid_table_len
[i
] = HNS_ROCE_V1_GID_NUM
/
902 for (i
= 0; i
< caps
->num_comp_vectors
; i
++)
903 caps
->ceqe_depth
[i
] = HNS_ROCE_V1_NUM_COMP_EQE
;
905 caps
->aeqe_depth
= HNS_ROCE_V1_NUM_ASYNC_EQE
;
906 caps
->local_ca_ack_delay
= le32_to_cpu(roce_read(hr_dev
,
907 ROCEE_ACK_DELAY_REG
));
908 caps
->max_mtu
= IB_MTU_2048
;
911 int hns_roce_v1_init(struct hns_roce_dev
*hr_dev
)
915 struct device
*dev
= &hr_dev
->pdev
->dev
;
917 /* DMAE user config */
918 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
);
919 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M
,
920 ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S
, 0xf);
921 roce_set_field(val
, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M
,
922 ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S
,
923 1 << PAGES_SHIFT_16
);
924 roce_write(hr_dev
, ROCEE_DMAE_USER_CFG1_REG
, val
);
926 val
= roce_read(hr_dev
, ROCEE_DMAE_USER_CFG2_REG
);
927 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M
,
928 ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S
, 0xf);
929 roce_set_field(val
, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M
,
930 ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S
,
931 1 << PAGES_SHIFT_16
);
933 ret
= hns_roce_db_init(hr_dev
);
935 dev_err(dev
, "doorbell init failed!\n");
939 ret
= hns_roce_raq_init(hr_dev
);
941 dev_err(dev
, "raq init failed!\n");
942 goto error_failed_raq_init
;
945 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_UP
);
949 error_failed_raq_init
:
950 hns_roce_db_free(hr_dev
);
954 void hns_roce_v1_exit(struct hns_roce_dev
*hr_dev
)
956 hns_roce_port_enable(hr_dev
, HNS_ROCE_PORT_DOWN
);
957 hns_roce_raq_free(hr_dev
);
958 hns_roce_db_free(hr_dev
);
961 void hns_roce_v1_set_gid(struct hns_roce_dev
*hr_dev
, u8 port
, int gid_index
,
967 gid_idx
= hns_get_gid_index(hr_dev
, port
, gid_index
);
969 p
= (u32
*)&gid
->raw
[0];
970 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_L_0_REG
+
971 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
973 p
= (u32
*)&gid
->raw
[4];
974 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_ML_0_REG
+
975 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
977 p
= (u32
*)&gid
->raw
[8];
978 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_MH_0_REG
+
979 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
981 p
= (u32
*)&gid
->raw
[0xc];
982 roce_raw_write(*p
, hr_dev
->reg_base
+ ROCEE_PORT_GID_H_0_REG
+
983 (HNS_ROCE_V1_GID_NUM
* gid_idx
));
986 void hns_roce_v1_set_mac(struct hns_roce_dev
*hr_dev
, u8 phy_port
, u8
*addr
)
994 p
= (u32
*)(&addr
[0]);
996 roce_raw_write(reg_smac_l
, hr_dev
->reg_base
+ ROCEE_SMAC_L_0_REG
+
997 PHY_PORT_OFFSET
* phy_port
);
999 val
= roce_read(hr_dev
,
1000 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1001 p_h
= (u16
*)(&addr
[4]);
1003 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_SMAC_H_M
,
1004 ROCEE_SMAC_H_ROCEE_SMAC_H_S
, reg_smac_h
);
1005 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1009 void hns_roce_v1_set_mtu(struct hns_roce_dev
*hr_dev
, u8 phy_port
,
1014 val
= roce_read(hr_dev
,
1015 ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
);
1016 roce_set_field(val
, ROCEE_SMAC_H_ROCEE_PORT_MTU_M
,
1017 ROCEE_SMAC_H_ROCEE_PORT_MTU_S
, mtu
);
1018 roce_write(hr_dev
, ROCEE_SMAC_H_0_REG
+ phy_port
* PHY_PORT_OFFSET
,
1022 int hns_roce_v1_write_mtpt(void *mb_buf
, struct hns_roce_mr
*mr
,
1023 unsigned long mtpt_idx
)
1025 struct hns_roce_v1_mpt_entry
*mpt_entry
;
1026 struct scatterlist
*sg
;
1031 /* MPT filled into mailbox buf */
1032 mpt_entry
= (struct hns_roce_v1_mpt_entry
*)mb_buf
;
1033 memset(mpt_entry
, 0, sizeof(*mpt_entry
));
1035 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_STATE_M
,
1036 MPT_BYTE_4_KEY_STATE_S
, KEY_VALID
);
1037 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_KEY_M
,
1038 MPT_BYTE_4_KEY_S
, mr
->key
);
1039 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_PAGE_SIZE_M
,
1040 MPT_BYTE_4_PAGE_SIZE_S
, MR_SIZE_4K
);
1041 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_TYPE_S
, 0);
1042 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MW_BIND_ENABLE_S
,
1043 (mr
->access
& IB_ACCESS_MW_BIND
? 1 : 0));
1044 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_OWN_S
, 0);
1045 roce_set_field(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M
,
1046 MPT_BYTE_4_MEMORY_LOCATION_TYPE_S
, mr
->type
);
1047 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_ATOMIC_S
, 0);
1048 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_LOCAL_WRITE_S
,
1049 (mr
->access
& IB_ACCESS_LOCAL_WRITE
? 1 : 0));
1050 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_WRITE_S
,
1051 (mr
->access
& IB_ACCESS_REMOTE_WRITE
? 1 : 0));
1052 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_READ_S
,
1053 (mr
->access
& IB_ACCESS_REMOTE_READ
? 1 : 0));
1054 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S
,
1056 roce_set_bit(mpt_entry
->mpt_byte_4
, MPT_BYTE_4_ADDRESS_TYPE_S
, 0);
1058 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1059 MPT_BYTE_12_PBL_ADDR_H_S
, 0);
1060 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_MW_BIND_COUNTER_M
,
1061 MPT_BYTE_12_MW_BIND_COUNTER_S
, 0);
1063 mpt_entry
->virt_addr_l
= (u32
)mr
->iova
;
1064 mpt_entry
->virt_addr_h
= (u32
)(mr
->iova
>> 32);
1065 mpt_entry
->length
= (u32
)mr
->size
;
1067 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_PD_M
,
1068 MPT_BYTE_28_PD_S
, mr
->pd
);
1069 roce_set_field(mpt_entry
->mpt_byte_28
, MPT_BYTE_28_L_KEY_IDX_L_M
,
1070 MPT_BYTE_28_L_KEY_IDX_L_S
, mtpt_idx
);
1071 roce_set_field(mpt_entry
->mpt_byte_64
, MPT_BYTE_64_L_KEY_IDX_H_M
,
1072 MPT_BYTE_64_L_KEY_IDX_H_S
, mtpt_idx
>> MTPT_IDX_SHIFT
);
1074 /* DMA momery regsiter */
1075 if (mr
->type
== MR_TYPE_DMA
)
1078 pages
= (u64
*) __get_free_page(GFP_KERNEL
);
1083 for_each_sg(mr
->umem
->sg_head
.sgl
, sg
, mr
->umem
->nmap
, entry
) {
1084 pages
[i
] = ((u64
)sg_dma_address(sg
)) >> 12;
1086 /* Directly record to MTPT table firstly 7 entry */
1087 if (i
>= HNS_ROCE_MAX_INNER_MTPT_NUM
)
1092 /* Register user mr */
1093 for (i
= 0; i
< HNS_ROCE_MAX_INNER_MTPT_NUM
; i
++) {
1096 mpt_entry
->pa0_l
= cpu_to_le32((u32
)(pages
[i
]));
1097 roce_set_field(mpt_entry
->mpt_byte_36
,
1098 MPT_BYTE_36_PA0_H_M
,
1099 MPT_BYTE_36_PA0_H_S
,
1100 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1103 roce_set_field(mpt_entry
->mpt_byte_36
,
1104 MPT_BYTE_36_PA1_L_M
,
1105 MPT_BYTE_36_PA1_L_S
,
1106 cpu_to_le32((u32
)(pages
[i
])));
1107 roce_set_field(mpt_entry
->mpt_byte_40
,
1108 MPT_BYTE_40_PA1_H_M
,
1109 MPT_BYTE_40_PA1_H_S
,
1110 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1113 roce_set_field(mpt_entry
->mpt_byte_40
,
1114 MPT_BYTE_40_PA2_L_M
,
1115 MPT_BYTE_40_PA2_L_S
,
1116 cpu_to_le32((u32
)(pages
[i
])));
1117 roce_set_field(mpt_entry
->mpt_byte_44
,
1118 MPT_BYTE_44_PA2_H_M
,
1119 MPT_BYTE_44_PA2_H_S
,
1120 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1123 roce_set_field(mpt_entry
->mpt_byte_44
,
1124 MPT_BYTE_44_PA3_L_M
,
1125 MPT_BYTE_44_PA3_L_S
,
1126 cpu_to_le32((u32
)(pages
[i
])));
1127 roce_set_field(mpt_entry
->mpt_byte_48
,
1128 MPT_BYTE_48_PA3_H_M
,
1129 MPT_BYTE_48_PA3_H_S
,
1130 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_8
)));
1133 mpt_entry
->pa4_l
= cpu_to_le32((u32
)(pages
[i
]));
1134 roce_set_field(mpt_entry
->mpt_byte_56
,
1135 MPT_BYTE_56_PA4_H_M
,
1136 MPT_BYTE_56_PA4_H_S
,
1137 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_32
)));
1140 roce_set_field(mpt_entry
->mpt_byte_56
,
1141 MPT_BYTE_56_PA5_L_M
,
1142 MPT_BYTE_56_PA5_L_S
,
1143 cpu_to_le32((u32
)(pages
[i
])));
1144 roce_set_field(mpt_entry
->mpt_byte_60
,
1145 MPT_BYTE_60_PA5_H_M
,
1146 MPT_BYTE_60_PA5_H_S
,
1147 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_24
)));
1150 roce_set_field(mpt_entry
->mpt_byte_60
,
1151 MPT_BYTE_60_PA6_L_M
,
1152 MPT_BYTE_60_PA6_L_S
,
1153 cpu_to_le32((u32
)(pages
[i
])));
1154 roce_set_field(mpt_entry
->mpt_byte_64
,
1155 MPT_BYTE_64_PA6_H_M
,
1156 MPT_BYTE_64_PA6_H_S
,
1157 cpu_to_le32((u32
)(pages
[i
] >> PAGES_SHIFT_16
)));
1164 free_page((unsigned long) pages
);
1166 mpt_entry
->pbl_addr_l
= (u32
)(mr
->pbl_dma_addr
);
1168 roce_set_field(mpt_entry
->mpt_byte_12
, MPT_BYTE_12_PBL_ADDR_H_M
,
1169 MPT_BYTE_12_PBL_ADDR_H_S
,
1170 ((u32
)(mr
->pbl_dma_addr
>> 32)));
1175 static void *get_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1177 return hns_roce_buf_offset(&hr_cq
->hr_buf
.hr_buf
,
1178 n
* HNS_ROCE_V1_CQE_ENTRY_SIZE
);
1181 static void *get_sw_cqe(struct hns_roce_cq
*hr_cq
, int n
)
1183 struct hns_roce_cqe
*hr_cqe
= get_cqe(hr_cq
, n
& hr_cq
->ib_cq
.cqe
);
1185 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1186 return (roce_get_bit(hr_cqe
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
) ^
1187 !!(n
& (hr_cq
->ib_cq
.cqe
+ 1))) ? hr_cqe
: NULL
;
1190 static struct hns_roce_cqe
*next_cqe_sw(struct hns_roce_cq
*hr_cq
)
1192 return get_sw_cqe(hr_cq
, hr_cq
->cons_index
);
1195 void hns_roce_v1_cq_set_ci(struct hns_roce_cq
*hr_cq
, u32 cons_index
,
1196 spinlock_t
*doorbell_lock
)
1201 doorbell
[0] = cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
1202 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
1203 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
1204 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
1205 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
1206 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 0);
1207 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
1208 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
, hr_cq
->cqn
);
1210 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
1213 static void __hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1214 struct hns_roce_srq
*srq
)
1216 struct hns_roce_cqe
*cqe
, *dest
;
1221 for (prod_index
= hr_cq
->cons_index
; get_sw_cqe(hr_cq
, prod_index
);
1223 if (prod_index
== hr_cq
->cons_index
+ hr_cq
->ib_cq
.cqe
)
1228 * Now backwards through the CQ, removing CQ entries
1229 * that match our QP by overwriting them with next entries.
1231 while ((int) --prod_index
- (int) hr_cq
->cons_index
>= 0) {
1232 cqe
= get_cqe(hr_cq
, prod_index
& hr_cq
->ib_cq
.cqe
);
1233 if ((roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1234 CQE_BYTE_16_LOCAL_QPN_S
) &
1235 HNS_ROCE_CQE_QPN_MASK
) == qpn
) {
1236 /* In v1 engine, not support SRQ */
1238 } else if (nfreed
) {
1239 dest
= get_cqe(hr_cq
, (prod_index
+ nfreed
) &
1241 owner_bit
= roce_get_bit(dest
->cqe_byte_4
,
1242 CQE_BYTE_4_OWNER_S
);
1243 memcpy(dest
, cqe
, sizeof(*cqe
));
1244 roce_set_bit(dest
->cqe_byte_4
, CQE_BYTE_4_OWNER_S
,
1250 hr_cq
->cons_index
+= nfreed
;
1252 * Make sure update of buffer contents is done before
1253 * updating consumer index.
1257 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
,
1258 &to_hr_dev(hr_cq
->ib_cq
.device
)->cq_db_lock
);
1262 static void hns_roce_v1_cq_clean(struct hns_roce_cq
*hr_cq
, u32 qpn
,
1263 struct hns_roce_srq
*srq
)
1265 spin_lock_irq(&hr_cq
->lock
);
1266 __hns_roce_v1_cq_clean(hr_cq
, qpn
, srq
);
1267 spin_unlock_irq(&hr_cq
->lock
);
1270 void hns_roce_v1_write_cqc(struct hns_roce_dev
*hr_dev
,
1271 struct hns_roce_cq
*hr_cq
, void *mb_buf
, u64
*mtts
,
1272 dma_addr_t dma_handle
, int nent
, u32 vector
)
1274 struct hns_roce_cq_context
*cq_context
= NULL
;
1275 void __iomem
*tptr_addr
;
1277 cq_context
= mb_buf
;
1278 memset(cq_context
, 0, sizeof(*cq_context
));
1281 hr_dev
->priv_addr
= tptr_addr
;
1282 hr_cq
->tptr_addr
= tptr_addr
;
1284 /* Register cq_context members */
1285 roce_set_field(cq_context
->cqc_byte_4
,
1286 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M
,
1287 CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S
, CQ_STATE_VALID
);
1288 roce_set_field(cq_context
->cqc_byte_4
, CQ_CONTEXT_CQC_BYTE_4_CQN_M
,
1289 CQ_CONTEXT_CQC_BYTE_4_CQN_S
, hr_cq
->cqn
);
1290 cq_context
->cqc_byte_4
= cpu_to_le32(cq_context
->cqc_byte_4
);
1292 cq_context
->cq_bt_l
= (u32
)dma_handle
;
1293 cq_context
->cq_bt_l
= cpu_to_le32(cq_context
->cq_bt_l
);
1295 roce_set_field(cq_context
->cqc_byte_12
,
1296 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M
,
1297 CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S
,
1298 ((u64
)dma_handle
>> 32));
1299 roce_set_field(cq_context
->cqc_byte_12
,
1300 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M
,
1301 CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S
,
1302 ilog2((unsigned int)nent
));
1303 roce_set_field(cq_context
->cqc_byte_12
, CQ_CONTEXT_CQC_BYTE_12_CEQN_M
,
1304 CQ_CONTEXT_CQC_BYTE_12_CEQN_S
, vector
);
1305 cq_context
->cqc_byte_12
= cpu_to_le32(cq_context
->cqc_byte_12
);
1307 cq_context
->cur_cqe_ba0_l
= (u32
)(mtts
[0]);
1308 cq_context
->cur_cqe_ba0_l
= cpu_to_le32(cq_context
->cur_cqe_ba0_l
);
1310 roce_set_field(cq_context
->cqc_byte_20
,
1311 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M
,
1312 CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S
,
1313 cpu_to_le32((mtts
[0]) >> 32));
1314 /* Dedicated hardware, directly set 0 */
1315 roce_set_field(cq_context
->cqc_byte_20
,
1316 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M
,
1317 CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S
, 0);
1319 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1320 * using 4K page, and shift more 32 because of
1321 * caculating the high 32 bit value evaluated to hardware.
1323 roce_set_field(cq_context
->cqc_byte_20
,
1324 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M
,
1325 CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S
,
1326 (u64
)tptr_addr
>> 44);
1327 cq_context
->cqc_byte_20
= cpu_to_le32(cq_context
->cqc_byte_20
);
1329 cq_context
->cqe_tptr_addr_l
= (u32
)((u64
)tptr_addr
>> 12);
1331 roce_set_field(cq_context
->cqc_byte_32
,
1332 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M
,
1333 CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S
, 0);
1334 roce_set_bit(cq_context
->cqc_byte_32
,
1335 CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S
, 0);
1336 roce_set_bit(cq_context
->cqc_byte_32
,
1337 CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S
, 0);
1338 roce_set_bit(cq_context
->cqc_byte_32
,
1339 CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S
, 0);
1340 roce_set_bit(cq_context
->cqc_byte_32
,
1341 CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S
,
1343 /*The initial value of cq's ci is 0 */
1344 roce_set_field(cq_context
->cqc_byte_32
,
1345 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M
,
1346 CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S
, 0);
1347 cq_context
->cqc_byte_32
= cpu_to_le32(cq_context
->cqc_byte_32
);
1350 int hns_roce_v1_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
1352 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
1353 u32 notification_flag
;
1357 notification_flag
= (flags
& IB_CQ_SOLICITED_MASK
) ==
1358 IB_CQ_SOLICITED
? CQ_DB_REQ_NOT
: CQ_DB_REQ_NOT_SOL
;
1360 * flags = 0; Notification Flag = 1, next
1361 * flags = 1; Notification Flag = 0, solocited
1363 doorbell
[0] = hr_cq
->cons_index
& ((hr_cq
->cq_depth
<< 1) - 1);
1364 roce_set_bit(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S
, 1);
1365 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M
,
1366 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S
, 3);
1367 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M
,
1368 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S
, 1);
1369 roce_set_field(doorbell
[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M
,
1370 ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S
,
1371 hr_cq
->cqn
| notification_flag
);
1373 hns_roce_write64_k(doorbell
, hr_cq
->cq_db_l
);
1378 static int hns_roce_v1_poll_one(struct hns_roce_cq
*hr_cq
,
1379 struct hns_roce_qp
**cur_qp
, struct ib_wc
*wc
)
1386 struct hns_roce_cqe
*cqe
;
1387 struct hns_roce_qp
*hr_qp
;
1388 struct hns_roce_wq
*wq
;
1389 struct hns_roce_wqe_ctrl_seg
*sq_wqe
;
1390 struct hns_roce_dev
*hr_dev
= to_hr_dev(hr_cq
->ib_cq
.device
);
1391 struct device
*dev
= &hr_dev
->pdev
->dev
;
1393 /* Find cqe according consumer index */
1394 cqe
= next_cqe_sw(hr_cq
);
1398 ++hr_cq
->cons_index
;
1399 /* Memory barrier */
1402 is_send
= !(roce_get_bit(cqe
->cqe_byte_4
, CQE_BYTE_4_SQ_RQ_FLAG_S
));
1404 /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
1405 if (roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1406 CQE_BYTE_16_LOCAL_QPN_S
) <= 1) {
1407 qpn
= roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_PORT_NUM_M
,
1408 CQE_BYTE_20_PORT_NUM_S
) +
1409 roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1410 CQE_BYTE_16_LOCAL_QPN_S
) *
1413 qpn
= roce_get_field(cqe
->cqe_byte_16
, CQE_BYTE_16_LOCAL_QPN_M
,
1414 CQE_BYTE_16_LOCAL_QPN_S
);
1417 if (!*cur_qp
|| (qpn
& HNS_ROCE_CQE_QPN_MASK
) != (*cur_qp
)->qpn
) {
1418 hr_qp
= __hns_roce_qp_lookup(hr_dev
, qpn
);
1419 if (unlikely(!hr_qp
)) {
1420 dev_err(dev
, "CQ %06lx with entry for unknown QPN %06x\n",
1421 hr_cq
->cqn
, (qpn
& HNS_ROCE_CQE_QPN_MASK
));
1428 wc
->qp
= &(*cur_qp
)->ibqp
;
1431 status
= roce_get_field(cqe
->cqe_byte_4
,
1432 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M
,
1433 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S
) &
1434 HNS_ROCE_CQE_STATUS_MASK
;
1436 case HNS_ROCE_CQE_SUCCESS
:
1437 wc
->status
= IB_WC_SUCCESS
;
1439 case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
1440 wc
->status
= IB_WC_LOC_LEN_ERR
;
1442 case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
1443 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
1445 case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR
:
1446 wc
->status
= IB_WC_LOC_PROT_ERR
;
1448 case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR
:
1449 wc
->status
= IB_WC_WR_FLUSH_ERR
;
1451 case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR
:
1452 wc
->status
= IB_WC_MW_BIND_ERR
;
1454 case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR
:
1455 wc
->status
= IB_WC_BAD_RESP_ERR
;
1457 case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
1458 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
1460 case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
1461 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
1463 case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
1464 wc
->status
= IB_WC_REM_ACCESS_ERR
;
1466 case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR
:
1467 wc
->status
= IB_WC_REM_OP_ERR
;
1469 case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
1470 wc
->status
= IB_WC_RETRY_EXC_ERR
;
1472 case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
1473 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
1476 wc
->status
= IB_WC_GENERAL_ERR
;
1480 /* CQE status error, directly return */
1481 if (wc
->status
!= IB_WC_SUCCESS
)
1485 /* SQ conrespond to CQE */
1486 sq_wqe
= get_send_wqe(*cur_qp
, roce_get_field(cqe
->cqe_byte_4
,
1487 CQE_BYTE_4_WQE_INDEX_M
,
1488 CQE_BYTE_4_WQE_INDEX_S
));
1489 switch (sq_wqe
->flag
& HNS_ROCE_WQE_OPCODE_MASK
) {
1490 case HNS_ROCE_WQE_OPCODE_SEND
:
1491 wc
->opcode
= IB_WC_SEND
;
1493 case HNS_ROCE_WQE_OPCODE_RDMA_READ
:
1494 wc
->opcode
= IB_WC_RDMA_READ
;
1495 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
1497 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE
:
1498 wc
->opcode
= IB_WC_RDMA_WRITE
;
1500 case HNS_ROCE_WQE_OPCODE_LOCAL_INV
:
1501 wc
->opcode
= IB_WC_LOCAL_INV
;
1503 case HNS_ROCE_WQE_OPCODE_UD_SEND
:
1504 wc
->opcode
= IB_WC_SEND
;
1507 wc
->status
= IB_WC_GENERAL_ERR
;
1510 wc
->wc_flags
= (sq_wqe
->flag
& HNS_ROCE_WQE_IMM
?
1511 IB_WC_WITH_IMM
: 0);
1513 wq
= &(*cur_qp
)->sq
;
1514 if ((*cur_qp
)->sq_signal_bits
) {
1516 * If sg_signal_bit is 1,
1517 * firstly tail pointer updated to wqe
1518 * which current cqe correspond to
1520 wqe_ctr
= (u16
)roce_get_field(cqe
->cqe_byte_4
,
1521 CQE_BYTE_4_WQE_INDEX_M
,
1522 CQE_BYTE_4_WQE_INDEX_S
);
1523 wq
->tail
+= (wqe_ctr
- (u16
)wq
->tail
) &
1526 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
1529 /* RQ conrespond to CQE */
1530 wc
->byte_len
= le32_to_cpu(cqe
->byte_cnt
);
1531 opcode
= roce_get_field(cqe
->cqe_byte_4
,
1532 CQE_BYTE_4_OPERATION_TYPE_M
,
1533 CQE_BYTE_4_OPERATION_TYPE_S
) &
1534 HNS_ROCE_CQE_OPCODE_MASK
;
1536 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE
:
1537 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
1538 wc
->wc_flags
= IB_WC_WITH_IMM
;
1539 wc
->ex
.imm_data
= le32_to_cpu(cqe
->immediate_data
);
1541 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE
:
1542 if (roce_get_bit(cqe
->cqe_byte_4
,
1543 CQE_BYTE_4_IMM_INDICATOR_S
)) {
1544 wc
->opcode
= IB_WC_RECV
;
1545 wc
->wc_flags
= IB_WC_WITH_IMM
;
1546 wc
->ex
.imm_data
= le32_to_cpu(
1547 cqe
->immediate_data
);
1549 wc
->opcode
= IB_WC_RECV
;
1554 wc
->status
= IB_WC_GENERAL_ERR
;
1558 /* Update tail pointer, record wr_id */
1559 wq
= &(*cur_qp
)->rq
;
1560 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
1562 wc
->sl
= (u8
)roce_get_field(cqe
->cqe_byte_20
, CQE_BYTE_20_SL_M
,
1564 wc
->src_qp
= (u8
)roce_get_field(cqe
->cqe_byte_20
,
1565 CQE_BYTE_20_REMOTE_QPN_M
,
1566 CQE_BYTE_20_REMOTE_QPN_S
);
1567 wc
->wc_flags
|= (roce_get_bit(cqe
->cqe_byte_20
,
1568 CQE_BYTE_20_GRH_PRESENT_S
) ?
1570 wc
->pkey_index
= (u16
)roce_get_field(cqe
->cqe_byte_28
,
1571 CQE_BYTE_28_P_KEY_IDX_M
,
1572 CQE_BYTE_28_P_KEY_IDX_S
);
1578 int hns_roce_v1_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
1580 struct hns_roce_cq
*hr_cq
= to_hr_cq(ibcq
);
1581 struct hns_roce_qp
*cur_qp
= NULL
;
1582 unsigned long flags
;
1586 spin_lock_irqsave(&hr_cq
->lock
, flags
);
1588 for (npolled
= 0; npolled
< num_entries
; ++npolled
) {
1589 ret
= hns_roce_v1_poll_one(hr_cq
, &cur_qp
, wc
+ npolled
);
1595 hns_roce_v1_cq_set_ci(hr_cq
, hr_cq
->cons_index
,
1596 &to_hr_dev(ibcq
->device
)->cq_db_lock
);
1599 spin_unlock_irqrestore(&hr_cq
->lock
, flags
);
1601 if (ret
== 0 || ret
== -EAGAIN
)
1607 static int hns_roce_v1_qp_modify(struct hns_roce_dev
*hr_dev
,
1608 struct hns_roce_mtt
*mtt
,
1609 enum hns_roce_qp_state cur_state
,
1610 enum hns_roce_qp_state new_state
,
1611 struct hns_roce_qp_context
*context
,
1612 struct hns_roce_qp
*hr_qp
)
1615 op
[HNS_ROCE_QP_NUM_STATE
][HNS_ROCE_QP_NUM_STATE
] = {
1616 [HNS_ROCE_QP_STATE_RST
] = {
1617 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1618 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1619 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
1621 [HNS_ROCE_QP_STATE_INIT
] = {
1622 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1623 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1624 /* Note: In v1 engine, HW doesn't support RST2INIT.
1625 * We use RST2INIT cmd instead of INIT2INIT.
1627 [HNS_ROCE_QP_STATE_INIT
] = HNS_ROCE_CMD_RST2INIT_QP
,
1628 [HNS_ROCE_QP_STATE_RTR
] = HNS_ROCE_CMD_INIT2RTR_QP
,
1630 [HNS_ROCE_QP_STATE_RTR
] = {
1631 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1632 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1633 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTR2RTS_QP
,
1635 [HNS_ROCE_QP_STATE_RTS
] = {
1636 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1637 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1638 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_RTS2RTS_QP
,
1639 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_RTS2SQD_QP
,
1641 [HNS_ROCE_QP_STATE_SQD
] = {
1642 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1643 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1644 [HNS_ROCE_QP_STATE_RTS
] = HNS_ROCE_CMD_SQD2RTS_QP
,
1645 [HNS_ROCE_QP_STATE_SQD
] = HNS_ROCE_CMD_SQD2SQD_QP
,
1647 [HNS_ROCE_QP_STATE_ERR
] = {
1648 [HNS_ROCE_QP_STATE_RST
] = HNS_ROCE_CMD_2RST_QP
,
1649 [HNS_ROCE_QP_STATE_ERR
] = HNS_ROCE_CMD_2ERR_QP
,
1653 struct hns_roce_cmd_mailbox
*mailbox
;
1654 struct device
*dev
= &hr_dev
->pdev
->dev
;
1657 if (cur_state
>= HNS_ROCE_QP_NUM_STATE
||
1658 new_state
>= HNS_ROCE_QP_NUM_STATE
||
1659 !op
[cur_state
][new_state
]) {
1660 dev_err(dev
, "[modify_qp]not support state %d to %d\n",
1661 cur_state
, new_state
);
1665 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2RST_QP
)
1666 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
1667 HNS_ROCE_CMD_2RST_QP
,
1668 HNS_ROCE_CMD_TIME_CLASS_A
);
1670 if (op
[cur_state
][new_state
] == HNS_ROCE_CMD_2ERR_QP
)
1671 return hns_roce_cmd_mbox(hr_dev
, 0, 0, hr_qp
->qpn
, 2,
1672 HNS_ROCE_CMD_2ERR_QP
,
1673 HNS_ROCE_CMD_TIME_CLASS_A
);
1675 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
1676 if (IS_ERR(mailbox
))
1677 return PTR_ERR(mailbox
);
1679 memcpy(mailbox
->buf
, context
, sizeof(*context
));
1681 ret
= hns_roce_cmd_mbox(hr_dev
, mailbox
->dma
, 0, hr_qp
->qpn
, 0,
1682 op
[cur_state
][new_state
],
1683 HNS_ROCE_CMD_TIME_CLASS_C
);
1685 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
1689 static int hns_roce_v1_m_sqp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
1690 int attr_mask
, enum ib_qp_state cur_state
,
1691 enum ib_qp_state new_state
)
1693 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
1694 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
1695 struct hns_roce_sqp_context
*context
;
1696 struct device
*dev
= &hr_dev
->pdev
->dev
;
1697 dma_addr_t dma_handle
= 0;
1703 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
1707 /* Search QP buf's MTTs */
1708 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
1709 hr_qp
->mtt
.first_seg
, &dma_handle
);
1711 dev_err(dev
, "qp buf pa find failed\n");
1715 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
1716 roce_set_field(context
->qp1c_bytes_4
,
1717 QP1C_BYTES_4_SQ_WQE_SHIFT_M
,
1718 QP1C_BYTES_4_SQ_WQE_SHIFT_S
,
1719 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
1720 roce_set_field(context
->qp1c_bytes_4
,
1721 QP1C_BYTES_4_RQ_WQE_SHIFT_M
,
1722 QP1C_BYTES_4_RQ_WQE_SHIFT_S
,
1723 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
1724 roce_set_field(context
->qp1c_bytes_4
, QP1C_BYTES_4_PD_M
,
1725 QP1C_BYTES_4_PD_S
, to_hr_pd(ibqp
->pd
)->pdn
);
1727 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
1728 roce_set_field(context
->qp1c_bytes_12
,
1729 QP1C_BYTES_12_SQ_RQ_BT_H_M
,
1730 QP1C_BYTES_12_SQ_RQ_BT_H_S
,
1731 ((u32
)(dma_handle
>> 32)));
1733 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_HEAD_M
,
1734 QP1C_BYTES_16_RQ_HEAD_S
, hr_qp
->rq
.head
);
1735 roce_set_field(context
->qp1c_bytes_16
, QP1C_BYTES_16_PORT_NUM_M
,
1736 QP1C_BYTES_16_PORT_NUM_S
, hr_qp
->port
);
1737 roce_set_bit(context
->qp1c_bytes_16
,
1738 QP1C_BYTES_16_SIGNALING_TYPE_S
,
1739 hr_qp
->sq_signal_bits
);
1740 roce_set_bit(context
->qp1c_bytes_16
,
1741 QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S
,
1742 hr_qp
->sq_signal_bits
);
1743 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_RQ_BA_FLG_S
,
1745 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_SQ_BA_FLG_S
,
1747 roce_set_bit(context
->qp1c_bytes_16
, QP1C_BYTES_16_QP1_ERR_S
,
1750 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_SQ_HEAD_M
,
1751 QP1C_BYTES_20_SQ_HEAD_S
, hr_qp
->sq
.head
);
1752 roce_set_field(context
->qp1c_bytes_20
, QP1C_BYTES_20_PKEY_IDX_M
,
1753 QP1C_BYTES_20_PKEY_IDX_S
, attr
->pkey_index
);
1755 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
1756 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
1758 roce_set_field(context
->qp1c_bytes_28
,
1759 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M
,
1760 QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S
,
1761 (mtts
[rq_pa_start
]) >> 32);
1762 roce_set_field(context
->qp1c_bytes_28
,
1763 QP1C_BYTES_28_RQ_CUR_IDX_M
,
1764 QP1C_BYTES_28_RQ_CUR_IDX_S
, 0);
1766 roce_set_field(context
->qp1c_bytes_32
,
1767 QP1C_BYTES_32_RX_CQ_NUM_M
,
1768 QP1C_BYTES_32_RX_CQ_NUM_S
,
1769 to_hr_cq(ibqp
->recv_cq
)->cqn
);
1770 roce_set_field(context
->qp1c_bytes_32
,
1771 QP1C_BYTES_32_TX_CQ_NUM_M
,
1772 QP1C_BYTES_32_TX_CQ_NUM_S
,
1773 to_hr_cq(ibqp
->send_cq
)->cqn
);
1775 context
->cur_sq_wqe_ba_l
= (u32
)mtts
[0];
1777 roce_set_field(context
->qp1c_bytes_40
,
1778 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M
,
1779 QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S
,
1781 roce_set_field(context
->qp1c_bytes_40
,
1782 QP1C_BYTES_40_SQ_CUR_IDX_M
,
1783 QP1C_BYTES_40_SQ_CUR_IDX_S
, 0);
1785 /* Copy context to QP1C register */
1786 addr
= (u32
*)(hr_dev
->reg_base
+ ROCEE_QP1C_CFG0_0_REG
+
1787 hr_qp
->port
* sizeof(*context
));
1789 writel(context
->qp1c_bytes_4
, addr
);
1790 writel(context
->sq_rq_bt_l
, addr
+ 1);
1791 writel(context
->qp1c_bytes_12
, addr
+ 2);
1792 writel(context
->qp1c_bytes_16
, addr
+ 3);
1793 writel(context
->qp1c_bytes_20
, addr
+ 4);
1794 writel(context
->cur_rq_wqe_ba_l
, addr
+ 5);
1795 writel(context
->qp1c_bytes_28
, addr
+ 6);
1796 writel(context
->qp1c_bytes_32
, addr
+ 7);
1797 writel(context
->cur_sq_wqe_ba_l
, addr
+ 8);
1800 /* Modify QP1C status */
1801 reg_val
= roce_read(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
1802 hr_qp
->port
* sizeof(*context
));
1803 roce_set_field(reg_val
, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M
,
1804 ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S
, new_state
);
1805 roce_write(hr_dev
, ROCEE_QP1C_CFG0_0_REG
+
1806 hr_qp
->port
* sizeof(*context
), reg_val
);
1808 hr_qp
->state
= new_state
;
1809 if (new_state
== IB_QPS_RESET
) {
1810 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
1811 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
1812 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
1813 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
1820 hr_qp
->sq_next_wqe
= 0;
1831 static int hns_roce_v1_m_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
1832 int attr_mask
, enum ib_qp_state cur_state
,
1833 enum ib_qp_state new_state
)
1835 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
1836 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
1837 struct device
*dev
= &hr_dev
->pdev
->dev
;
1838 struct hns_roce_qp_context
*context
;
1839 struct hns_roce_rq_db rq_db
;
1840 dma_addr_t dma_handle_2
= 0;
1841 dma_addr_t dma_handle
= 0;
1842 uint32_t doorbell
[2] = {0};
1843 int rq_pa_start
= 0;
1852 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
1856 /* Search qp buf's mtts */
1857 mtts
= hns_roce_table_find(&hr_dev
->mr_table
.mtt_table
,
1858 hr_qp
->mtt
.first_seg
, &dma_handle
);
1860 dev_err(dev
, "qp buf pa find failed\n");
1864 /* Search IRRL's mtts */
1865 mtts_2
= hns_roce_table_find(&hr_dev
->qp_table
.irrl_table
, hr_qp
->qpn
,
1867 if (mtts_2
== NULL
) {
1868 dev_err(dev
, "qp irrl_table find failed\n");
1875 * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
1876 * Optional param: NA
1878 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
1879 roce_set_field(context
->qpc_bytes_4
,
1880 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
1881 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
1882 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
1884 roce_set_bit(context
->qpc_bytes_4
,
1885 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
1886 roce_set_bit(context
->qpc_bytes_4
,
1887 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
1888 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
));
1889 roce_set_bit(context
->qpc_bytes_4
,
1890 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
1891 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
)
1893 roce_set_bit(context
->qpc_bytes_4
,
1894 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
,
1895 !!(attr
->qp_access_flags
& IB_ACCESS_REMOTE_ATOMIC
)
1897 roce_set_bit(context
->qpc_bytes_4
,
1898 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
1899 roce_set_field(context
->qpc_bytes_4
,
1900 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
1901 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
1902 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
1903 roce_set_field(context
->qpc_bytes_4
,
1904 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
1905 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
1906 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
1907 roce_set_field(context
->qpc_bytes_4
,
1908 QP_CONTEXT_QPC_BYTES_4_PD_M
,
1909 QP_CONTEXT_QPC_BYTES_4_PD_S
,
1910 to_hr_pd(ibqp
->pd
)->pdn
);
1911 hr_qp
->access_flags
= attr
->qp_access_flags
;
1912 roce_set_field(context
->qpc_bytes_8
,
1913 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
1914 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
1915 to_hr_cq(ibqp
->send_cq
)->cqn
);
1916 roce_set_field(context
->qpc_bytes_8
,
1917 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
1918 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
1919 to_hr_cq(ibqp
->recv_cq
)->cqn
);
1922 roce_set_field(context
->qpc_bytes_12
,
1923 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
1924 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
1925 to_hr_srq(ibqp
->srq
)->srqn
);
1927 roce_set_field(context
->qpc_bytes_12
,
1928 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
1929 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
1931 hr_qp
->pkey_index
= attr
->pkey_index
;
1932 roce_set_field(context
->qpc_bytes_16
,
1933 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
1934 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
1936 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
1937 roce_set_field(context
->qpc_bytes_4
,
1938 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M
,
1939 QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S
,
1940 to_hr_qp_type(hr_qp
->ibqp
.qp_type
));
1941 roce_set_bit(context
->qpc_bytes_4
,
1942 QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S
, 0);
1943 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
1944 roce_set_bit(context
->qpc_bytes_4
,
1945 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
1946 !!(attr
->qp_access_flags
&
1947 IB_ACCESS_REMOTE_READ
));
1948 roce_set_bit(context
->qpc_bytes_4
,
1949 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
1950 !!(attr
->qp_access_flags
&
1951 IB_ACCESS_REMOTE_WRITE
));
1953 roce_set_bit(context
->qpc_bytes_4
,
1954 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
,
1955 !!(hr_qp
->access_flags
&
1956 IB_ACCESS_REMOTE_READ
));
1957 roce_set_bit(context
->qpc_bytes_4
,
1958 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
,
1959 !!(hr_qp
->access_flags
&
1960 IB_ACCESS_REMOTE_WRITE
));
1963 roce_set_bit(context
->qpc_bytes_4
,
1964 QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S
, 1);
1965 roce_set_field(context
->qpc_bytes_4
,
1966 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M
,
1967 QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S
,
1968 ilog2((unsigned int)hr_qp
->sq
.wqe_cnt
));
1969 roce_set_field(context
->qpc_bytes_4
,
1970 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M
,
1971 QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S
,
1972 ilog2((unsigned int)hr_qp
->rq
.wqe_cnt
));
1973 roce_set_field(context
->qpc_bytes_4
,
1974 QP_CONTEXT_QPC_BYTES_4_PD_M
,
1975 QP_CONTEXT_QPC_BYTES_4_PD_S
,
1976 to_hr_pd(ibqp
->pd
)->pdn
);
1978 roce_set_field(context
->qpc_bytes_8
,
1979 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M
,
1980 QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S
,
1981 to_hr_cq(ibqp
->send_cq
)->cqn
);
1982 roce_set_field(context
->qpc_bytes_8
,
1983 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M
,
1984 QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S
,
1985 to_hr_cq(ibqp
->recv_cq
)->cqn
);
1988 roce_set_field(context
->qpc_bytes_12
,
1989 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M
,
1990 QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S
,
1991 to_hr_srq(ibqp
->srq
)->srqn
);
1992 if (attr_mask
& IB_QP_PKEY_INDEX
)
1993 roce_set_field(context
->qpc_bytes_12
,
1994 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
1995 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
1998 roce_set_field(context
->qpc_bytes_12
,
1999 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2000 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
,
2003 roce_set_field(context
->qpc_bytes_16
,
2004 QP_CONTEXT_QPC_BYTES_16_QP_NUM_M
,
2005 QP_CONTEXT_QPC_BYTES_16_QP_NUM_S
, hr_qp
->qpn
);
2006 } else if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RTR
) {
2007 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2008 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2009 (attr_mask
& IB_QP_PKEY_INDEX
) ||
2010 (attr_mask
& IB_QP_QKEY
)) {
2011 dev_err(dev
, "INIT2RTR attr_mask error\n");
2015 dmac
= (u8
*)attr
->ah_attr
.dmac
;
2017 context
->sq_rq_bt_l
= (u32
)(dma_handle
);
2018 roce_set_field(context
->qpc_bytes_24
,
2019 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M
,
2020 QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S
,
2021 ((u32
)(dma_handle
>> 32)));
2022 roce_set_bit(context
->qpc_bytes_24
,
2023 QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S
,
2025 roce_set_field(context
->qpc_bytes_24
,
2026 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
2027 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
,
2028 attr
->min_rnr_timer
);
2029 context
->irrl_ba_l
= (u32
)(dma_handle_2
);
2030 roce_set_field(context
->qpc_bytes_32
,
2031 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
,
2032 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S
,
2033 ((u32
)(dma_handle_2
>> 32)) &
2034 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M
);
2035 roce_set_field(context
->qpc_bytes_32
,
2036 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M
,
2037 QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S
, 0);
2038 roce_set_bit(context
->qpc_bytes_32
,
2039 QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S
,
2041 roce_set_bit(context
->qpc_bytes_32
,
2042 QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S
,
2043 hr_qp
->sq_signal_bits
);
2045 for (port
= 0; port
< hr_dev
->caps
.num_ports
; port
++) {
2046 smac
= (u8
*)hr_dev
->dev_addr
[port
];
2047 dev_dbg(dev
, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
2048 smac
[0], smac
[1], smac
[2], smac
[3], smac
[4],
2050 if ((dmac
[0] == smac
[0]) && (dmac
[1] == smac
[1]) &&
2051 (dmac
[2] == smac
[2]) && (dmac
[3] == smac
[3]) &&
2052 (dmac
[4] == smac
[4]) && (dmac
[5] == smac
[5])) {
2053 roce_set_bit(context
->qpc_bytes_32
,
2054 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S
,
2060 if (hr_dev
->loop_idc
== 0x1)
2061 roce_set_bit(context
->qpc_bytes_32
,
2062 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S
, 1);
2064 roce_set_bit(context
->qpc_bytes_32
,
2065 QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S
,
2066 attr
->ah_attr
.ah_flags
);
2067 roce_set_field(context
->qpc_bytes_32
,
2068 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
2069 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
,
2070 ilog2((unsigned int)attr
->max_dest_rd_atomic
));
2072 roce_set_field(context
->qpc_bytes_36
,
2073 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
2074 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
,
2077 /* Configure GID index */
2078 roce_set_field(context
->qpc_bytes_36
,
2079 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
2080 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
,
2081 hns_get_gid_index(hr_dev
,
2082 attr
->ah_attr
.port_num
- 1,
2083 attr
->ah_attr
.grh
.sgid_index
));
2085 memcpy(&(context
->dmac_l
), dmac
, 4);
2087 roce_set_field(context
->qpc_bytes_44
,
2088 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2089 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
,
2090 *((u16
*)(&dmac
[4])));
2091 roce_set_field(context
->qpc_bytes_44
,
2092 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M
,
2093 QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S
,
2094 attr
->ah_attr
.static_rate
);
2095 roce_set_field(context
->qpc_bytes_44
,
2096 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
2097 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
,
2098 attr
->ah_attr
.grh
.hop_limit
);
2100 roce_set_field(context
->qpc_bytes_48
,
2101 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
2102 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
,
2103 attr
->ah_attr
.grh
.flow_label
);
2104 roce_set_field(context
->qpc_bytes_48
,
2105 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
2106 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
,
2107 attr
->ah_attr
.grh
.traffic_class
);
2108 roce_set_field(context
->qpc_bytes_48
,
2109 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
2110 QP_CONTEXT_QPC_BYTES_48_MTU_S
, attr
->path_mtu
);
2112 memcpy(context
->dgid
, attr
->ah_attr
.grh
.dgid
.raw
,
2113 sizeof(attr
->ah_attr
.grh
.dgid
.raw
));
2115 dev_dbg(dev
, "dmac:%x :%lx\n", context
->dmac_l
,
2116 roce_get_field(context
->qpc_bytes_44
,
2117 QP_CONTEXT_QPC_BYTES_44_DMAC_H_M
,
2118 QP_CONTEXT_QPC_BYTES_44_DMAC_H_S
));
2120 roce_set_field(context
->qpc_bytes_68
,
2121 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M
,
2122 QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S
, 0);
2123 roce_set_field(context
->qpc_bytes_68
,
2124 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M
,
2125 QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S
, 0);
2127 rq_pa_start
= (u32
)hr_qp
->rq
.offset
/ PAGE_SIZE
;
2128 context
->cur_rq_wqe_ba_l
= (u32
)(mtts
[rq_pa_start
]);
2130 roce_set_field(context
->qpc_bytes_76
,
2131 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M
,
2132 QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S
,
2133 mtts
[rq_pa_start
] >> 32);
2134 roce_set_field(context
->qpc_bytes_76
,
2135 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M
,
2136 QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S
, 0);
2138 context
->rx_rnr_time
= 0;
2140 roce_set_field(context
->qpc_bytes_84
,
2141 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M
,
2142 QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S
,
2144 roce_set_field(context
->qpc_bytes_84
,
2145 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M
,
2146 QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S
, 0);
2148 roce_set_field(context
->qpc_bytes_88
,
2149 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
2150 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
,
2152 roce_set_bit(context
->qpc_bytes_88
,
2153 QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S
, 0);
2154 roce_set_bit(context
->qpc_bytes_88
,
2155 QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S
, 0);
2156 roce_set_field(context
->qpc_bytes_88
,
2157 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M
,
2158 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S
,
2160 roce_set_field(context
->qpc_bytes_88
,
2161 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M
,
2162 QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S
,
2165 context
->dma_length
= 0;
2170 roce_set_field(context
->qpc_bytes_108
,
2171 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M
,
2172 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S
, 0);
2173 roce_set_bit(context
->qpc_bytes_108
,
2174 QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S
, 0);
2175 roce_set_bit(context
->qpc_bytes_108
,
2176 QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S
, 0);
2178 roce_set_field(context
->qpc_bytes_112
,
2179 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M
,
2180 QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S
, 0);
2181 roce_set_field(context
->qpc_bytes_112
,
2182 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M
,
2183 QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S
, 0);
2185 /* For chip resp ack */
2186 roce_set_field(context
->qpc_bytes_156
,
2187 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
2188 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
2190 roce_set_field(context
->qpc_bytes_156
,
2191 QP_CONTEXT_QPC_BYTES_156_SL_M
,
2192 QP_CONTEXT_QPC_BYTES_156_SL_S
, attr
->ah_attr
.sl
);
2193 hr_qp
->sl
= attr
->ah_attr
.sl
;
2194 } else if (cur_state
== IB_QPS_RTR
&&
2195 new_state
== IB_QPS_RTS
) {
2196 /* If exist optional param, return error */
2197 if ((attr_mask
& IB_QP_ALT_PATH
) ||
2198 (attr_mask
& IB_QP_ACCESS_FLAGS
) ||
2199 (attr_mask
& IB_QP_QKEY
) ||
2200 (attr_mask
& IB_QP_PATH_MIG_STATE
) ||
2201 (attr_mask
& IB_QP_CUR_STATE
) ||
2202 (attr_mask
& IB_QP_MIN_RNR_TIMER
)) {
2203 dev_err(dev
, "RTR2RTS attr_mask error\n");
2207 context
->rx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
2209 roce_set_field(context
->qpc_bytes_120
,
2210 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M
,
2211 QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S
,
2214 roce_set_field(context
->qpc_bytes_124
,
2215 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M
,
2216 QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S
, 0);
2217 roce_set_field(context
->qpc_bytes_124
,
2218 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M
,
2219 QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S
, 0);
2221 roce_set_field(context
->qpc_bytes_128
,
2222 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M
,
2223 QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S
,
2225 roce_set_bit(context
->qpc_bytes_128
,
2226 QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S
, 0);
2227 roce_set_field(context
->qpc_bytes_128
,
2228 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M
,
2229 QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S
,
2231 roce_set_bit(context
->qpc_bytes_128
,
2232 QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S
, 0);
2234 roce_set_field(context
->qpc_bytes_132
,
2235 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M
,
2236 QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S
, 0);
2237 roce_set_field(context
->qpc_bytes_132
,
2238 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M
,
2239 QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S
, 0);
2241 roce_set_field(context
->qpc_bytes_136
,
2242 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M
,
2243 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S
,
2245 roce_set_field(context
->qpc_bytes_136
,
2246 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M
,
2247 QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S
,
2250 roce_set_field(context
->qpc_bytes_140
,
2251 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M
,
2252 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S
,
2253 (attr
->sq_psn
>> SQ_PSN_SHIFT
));
2254 roce_set_field(context
->qpc_bytes_140
,
2255 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M
,
2256 QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S
, 0);
2257 roce_set_bit(context
->qpc_bytes_140
,
2258 QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S
, 0);
2260 roce_set_field(context
->qpc_bytes_144
,
2261 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
2262 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
,
2265 roce_set_field(context
->qpc_bytes_148
,
2266 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M
,
2267 QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S
, 0);
2268 roce_set_field(context
->qpc_bytes_148
,
2269 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
2270 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
, 0);
2271 roce_set_field(context
->qpc_bytes_148
,
2272 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M
,
2273 QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S
, 0);
2274 roce_set_field(context
->qpc_bytes_148
,
2275 QP_CONTEXT_QPC_BYTES_148_LSN_M
,
2276 QP_CONTEXT_QPC_BYTES_148_LSN_S
, 0x100);
2278 context
->rnr_retry
= 0;
2280 roce_set_field(context
->qpc_bytes_156
,
2281 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M
,
2282 QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S
,
2284 roce_set_field(context
->qpc_bytes_156
,
2285 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
2286 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
,
2288 roce_set_field(context
->qpc_bytes_156
,
2289 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M
,
2290 QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S
,
2292 roce_set_field(context
->qpc_bytes_156
,
2293 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
2294 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
,
2296 roce_set_field(context
->qpc_bytes_156
,
2297 QP_CONTEXT_QPC_BYTES_156_SL_M
,
2298 QP_CONTEXT_QPC_BYTES_156_SL_S
, attr
->ah_attr
.sl
);
2299 hr_qp
->sl
= attr
->ah_attr
.sl
;
2300 roce_set_field(context
->qpc_bytes_156
,
2301 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
2302 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
,
2303 ilog2((unsigned int)attr
->max_rd_atomic
));
2304 roce_set_field(context
->qpc_bytes_156
,
2305 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M
,
2306 QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S
, 0);
2307 context
->pkt_use_len
= 0;
2309 roce_set_field(context
->qpc_bytes_164
,
2310 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
2311 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
, attr
->sq_psn
);
2312 roce_set_field(context
->qpc_bytes_164
,
2313 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M
,
2314 QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S
, 0);
2316 roce_set_field(context
->qpc_bytes_168
,
2317 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M
,
2318 QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S
,
2320 roce_set_field(context
->qpc_bytes_168
,
2321 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M
,
2322 QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S
, 0);
2323 roce_set_field(context
->qpc_bytes_168
,
2324 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M
,
2325 QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S
, 0);
2326 roce_set_bit(context
->qpc_bytes_168
,
2327 QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S
, 0);
2328 roce_set_bit(context
->qpc_bytes_168
,
2329 QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S
, 0);
2330 roce_set_bit(context
->qpc_bytes_168
,
2331 QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S
, 0);
2332 context
->sge_use_len
= 0;
2334 roce_set_field(context
->qpc_bytes_176
,
2335 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M
,
2336 QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S
, 0);
2337 roce_set_field(context
->qpc_bytes_176
,
2338 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M
,
2339 QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S
,
2341 roce_set_field(context
->qpc_bytes_180
,
2342 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M
,
2343 QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S
, 0);
2344 roce_set_field(context
->qpc_bytes_180
,
2345 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M
,
2346 QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S
, 0);
2348 context
->tx_cur_sq_wqe_ba_l
= (u32
)(mtts
[0]);
2350 roce_set_field(context
->qpc_bytes_188
,
2351 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M
,
2352 QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S
,
2354 roce_set_bit(context
->qpc_bytes_188
,
2355 QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S
, 0);
2356 roce_set_field(context
->qpc_bytes_188
,
2357 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M
,
2358 QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S
,
2360 } else if ((cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_RESET
) ||
2361 (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_ERR
) ||
2362 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_RESET
) ||
2363 (cur_state
== IB_QPS_RTR
&& new_state
== IB_QPS_ERR
) ||
2364 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_RESET
) ||
2365 (cur_state
== IB_QPS_RTS
&& new_state
== IB_QPS_ERR
) ||
2366 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_RESET
) ||
2367 (cur_state
== IB_QPS_ERR
&& new_state
== IB_QPS_ERR
)) {
2368 roce_set_field(context
->qpc_bytes_144
,
2369 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
2370 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
,
2374 dev_err(dev
, "not support this modify\n");
2378 /* Every status migrate must change state */
2379 roce_set_field(context
->qpc_bytes_144
,
2380 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
2381 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
, attr
->qp_state
);
2383 /* SW pass context to HW */
2384 ret
= hns_roce_v1_qp_modify(hr_dev
, &hr_qp
->mtt
,
2385 to_hns_roce_state(cur_state
),
2386 to_hns_roce_state(new_state
), context
,
2389 dev_err(dev
, "hns_roce_qp_modify failed\n");
2394 * Use rst2init to instead of init2init with drv,
2395 * need to hw to flash RQ HEAD by DB again
2397 if (cur_state
== IB_QPS_INIT
&& new_state
== IB_QPS_INIT
) {
2398 /* Memory barrier */
2400 if (hr_qp
->ibqp
.qp_type
== IB_QPT_GSI
) {
2401 /* SW update GSI rq header */
2402 reg_val
= roce_read(hr_dev
, ROCEE_QP1C_CFG3_0_REG
+
2403 QP1C_CFGN_OFFSET
* hr_qp
->port
);
2404 roce_set_field(reg_val
,
2405 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M
,
2406 ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S
,
2408 roce_write(hr_dev
, ROCEE_QP1C_CFG3_0_REG
+
2409 QP1C_CFGN_OFFSET
* hr_qp
->port
, reg_val
);
2414 roce_set_field(rq_db
.u32_4
, RQ_DOORBELL_U32_4_RQ_HEAD_M
,
2415 RQ_DOORBELL_U32_4_RQ_HEAD_S
,
2417 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_QPN_M
,
2418 RQ_DOORBELL_U32_8_QPN_S
, hr_qp
->qpn
);
2419 roce_set_field(rq_db
.u32_8
, RQ_DOORBELL_U32_8_CMD_M
,
2420 RQ_DOORBELL_U32_8_CMD_S
, 1);
2421 roce_set_bit(rq_db
.u32_8
, RQ_DOORBELL_U32_8_HW_SYNC_S
,
2424 doorbell
[0] = rq_db
.u32_4
;
2425 doorbell
[1] = rq_db
.u32_8
;
2427 hns_roce_write64_k(doorbell
, hr_qp
->rq
.db_reg_l
);
2431 hr_qp
->state
= new_state
;
2433 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
2434 hr_qp
->resp_depth
= attr
->max_dest_rd_atomic
;
2435 if (attr_mask
& IB_QP_PORT
)
2436 hr_qp
->port
= (attr
->port_num
- 1);
2438 if (new_state
== IB_QPS_RESET
&& !ibqp
->uobject
) {
2439 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->recv_cq
), hr_qp
->qpn
,
2440 ibqp
->srq
? to_hr_srq(ibqp
->srq
) : NULL
);
2441 if (ibqp
->send_cq
!= ibqp
->recv_cq
)
2442 hns_roce_v1_cq_clean(to_hr_cq(ibqp
->send_cq
),
2449 hr_qp
->sq_next_wqe
= 0;
2456 int hns_roce_v1_modify_qp(struct ib_qp
*ibqp
, const struct ib_qp_attr
*attr
,
2457 int attr_mask
, enum ib_qp_state cur_state
,
2458 enum ib_qp_state new_state
)
2461 if (ibqp
->qp_type
== IB_QPT_GSI
|| ibqp
->qp_type
== IB_QPT_SMI
)
2462 return hns_roce_v1_m_sqp(ibqp
, attr
, attr_mask
, cur_state
,
2465 return hns_roce_v1_m_qp(ibqp
, attr
, attr_mask
, cur_state
,
2469 static enum ib_qp_state
to_ib_qp_state(enum hns_roce_qp_state state
)
2472 case HNS_ROCE_QP_STATE_RST
:
2473 return IB_QPS_RESET
;
2474 case HNS_ROCE_QP_STATE_INIT
:
2476 case HNS_ROCE_QP_STATE_RTR
:
2478 case HNS_ROCE_QP_STATE_RTS
:
2480 case HNS_ROCE_QP_STATE_SQD
:
2482 case HNS_ROCE_QP_STATE_ERR
:
2489 static int hns_roce_v1_query_qpc(struct hns_roce_dev
*hr_dev
,
2490 struct hns_roce_qp
*hr_qp
,
2491 struct hns_roce_qp_context
*hr_context
)
2493 struct hns_roce_cmd_mailbox
*mailbox
;
2496 mailbox
= hns_roce_alloc_cmd_mailbox(hr_dev
);
2497 if (IS_ERR(mailbox
))
2498 return PTR_ERR(mailbox
);
2500 ret
= hns_roce_cmd_mbox(hr_dev
, 0, mailbox
->dma
, hr_qp
->qpn
, 0,
2501 HNS_ROCE_CMD_QUERY_QP
,
2502 HNS_ROCE_CMD_TIME_CLASS_A
);
2504 memcpy(hr_context
, mailbox
->buf
, sizeof(*hr_context
));
2506 dev_err(&hr_dev
->pdev
->dev
, "QUERY QP cmd process error\n");
2508 hns_roce_free_cmd_mailbox(hr_dev
, mailbox
);
2513 int hns_roce_v1_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
2514 int qp_attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
2516 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2517 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2518 struct device
*dev
= &hr_dev
->pdev
->dev
;
2519 struct hns_roce_qp_context
*context
;
2520 int tmp_qp_state
= 0;
2524 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2528 memset(qp_attr
, 0, sizeof(*qp_attr
));
2529 memset(qp_init_attr
, 0, sizeof(*qp_init_attr
));
2531 mutex_lock(&hr_qp
->mutex
);
2533 if (hr_qp
->state
== IB_QPS_RESET
) {
2534 qp_attr
->qp_state
= IB_QPS_RESET
;
2538 ret
= hns_roce_v1_query_qpc(hr_dev
, hr_qp
, context
);
2540 dev_err(dev
, "query qpc error\n");
2545 state
= roce_get_field(context
->qpc_bytes_144
,
2546 QP_CONTEXT_QPC_BYTES_144_QP_STATE_M
,
2547 QP_CONTEXT_QPC_BYTES_144_QP_STATE_S
);
2548 tmp_qp_state
= (int)to_ib_qp_state((enum hns_roce_qp_state
)state
);
2549 if (tmp_qp_state
== -1) {
2550 dev_err(dev
, "to_ib_qp_state error\n");
2554 hr_qp
->state
= (u8
)tmp_qp_state
;
2555 qp_attr
->qp_state
= (enum ib_qp_state
)hr_qp
->state
;
2556 qp_attr
->path_mtu
= (enum ib_mtu
)roce_get_field(context
->qpc_bytes_48
,
2557 QP_CONTEXT_QPC_BYTES_48_MTU_M
,
2558 QP_CONTEXT_QPC_BYTES_48_MTU_S
);
2559 qp_attr
->path_mig_state
= IB_MIG_ARMED
;
2560 if (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
)
2561 qp_attr
->qkey
= QKEY_VAL
;
2563 qp_attr
->rq_psn
= roce_get_field(context
->qpc_bytes_88
,
2564 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M
,
2565 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S
);
2566 qp_attr
->sq_psn
= (u32
)roce_get_field(context
->qpc_bytes_164
,
2567 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M
,
2568 QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S
);
2569 qp_attr
->dest_qp_num
= (u8
)roce_get_field(context
->qpc_bytes_36
,
2570 QP_CONTEXT_QPC_BYTES_36_DEST_QP_M
,
2571 QP_CONTEXT_QPC_BYTES_36_DEST_QP_S
);
2572 qp_attr
->qp_access_flags
= ((roce_get_bit(context
->qpc_bytes_4
,
2573 QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S
)) << 2) |
2574 ((roce_get_bit(context
->qpc_bytes_4
,
2575 QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S
)) << 1) |
2576 ((roce_get_bit(context
->qpc_bytes_4
,
2577 QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S
)) << 3);
2579 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
||
2580 hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) {
2581 qp_attr
->ah_attr
.sl
= roce_get_field(context
->qpc_bytes_156
,
2582 QP_CONTEXT_QPC_BYTES_156_SL_M
,
2583 QP_CONTEXT_QPC_BYTES_156_SL_S
);
2584 qp_attr
->ah_attr
.grh
.flow_label
= roce_get_field(
2585 context
->qpc_bytes_48
,
2586 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M
,
2587 QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S
);
2588 qp_attr
->ah_attr
.grh
.sgid_index
= roce_get_field(
2589 context
->qpc_bytes_36
,
2590 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M
,
2591 QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S
);
2592 qp_attr
->ah_attr
.grh
.hop_limit
= roce_get_field(
2593 context
->qpc_bytes_44
,
2594 QP_CONTEXT_QPC_BYTES_44_HOPLMT_M
,
2595 QP_CONTEXT_QPC_BYTES_44_HOPLMT_S
);
2596 qp_attr
->ah_attr
.grh
.traffic_class
= roce_get_field(
2597 context
->qpc_bytes_48
,
2598 QP_CONTEXT_QPC_BYTES_48_TCLASS_M
,
2599 QP_CONTEXT_QPC_BYTES_48_TCLASS_S
);
2601 memcpy(qp_attr
->ah_attr
.grh
.dgid
.raw
, context
->dgid
,
2602 sizeof(qp_attr
->ah_attr
.grh
.dgid
.raw
));
2605 qp_attr
->pkey_index
= roce_get_field(context
->qpc_bytes_12
,
2606 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M
,
2607 QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S
);
2608 qp_attr
->port_num
= (u8
)roce_get_field(context
->qpc_bytes_156
,
2609 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M
,
2610 QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S
) + 1;
2611 qp_attr
->sq_draining
= 0;
2612 qp_attr
->max_rd_atomic
= roce_get_field(context
->qpc_bytes_156
,
2613 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M
,
2614 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S
);
2615 qp_attr
->max_dest_rd_atomic
= roce_get_field(context
->qpc_bytes_32
,
2616 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M
,
2617 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S
);
2618 qp_attr
->min_rnr_timer
= (u8
)(roce_get_field(context
->qpc_bytes_24
,
2619 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M
,
2620 QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S
));
2621 qp_attr
->timeout
= (u8
)(roce_get_field(context
->qpc_bytes_156
,
2622 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M
,
2623 QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S
));
2624 qp_attr
->retry_cnt
= roce_get_field(context
->qpc_bytes_148
,
2625 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M
,
2626 QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S
);
2627 qp_attr
->rnr_retry
= context
->rnr_retry
;
2630 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
2631 qp_attr
->cap
.max_recv_wr
= hr_qp
->rq
.wqe_cnt
;
2632 qp_attr
->cap
.max_recv_sge
= hr_qp
->rq
.max_gs
;
2634 if (!ibqp
->uobject
) {
2635 qp_attr
->cap
.max_send_wr
= hr_qp
->sq
.wqe_cnt
;
2636 qp_attr
->cap
.max_send_sge
= hr_qp
->sq
.max_gs
;
2638 qp_attr
->cap
.max_send_wr
= 0;
2639 qp_attr
->cap
.max_send_sge
= 0;
2642 qp_init_attr
->cap
= qp_attr
->cap
;
2645 mutex_unlock(&hr_qp
->mutex
);
2650 static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev
*hr_dev
,
2651 struct hns_roce_qp
*hr_qp
,
2655 unsigned long end
= 0;
2659 struct hns_roce_cq
*send_cq
, *recv_cq
;
2660 struct device
*dev
= &hr_dev
->pdev
->dev
;
2662 if (hr_qp
->ibqp
.qp_type
== IB_QPT_RC
) {
2663 if (hr_qp
->state
!= IB_QPS_RESET
) {
2666 * waiting for hw complete processing all dbs
2668 if (hns_roce_v1_qp_modify(hr_dev
, NULL
,
2670 (enum ib_qp_state
)hr_qp
->state
),
2671 HNS_ROCE_QP_STATE_ERR
, NULL
,
2673 dev_err(dev
, "modify QP %06lx to ERR failed.\n",
2676 /* Record issued doorbell */
2677 sdbisusepr_val
= roce_read(hr_dev
,
2678 ROCEE_SDB_ISSUE_PTR_REG
);
2680 * Query db process status,
2681 * until hw process completely
2683 end
= msecs_to_jiffies(
2684 HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS
) + jiffies
;
2686 sdbsendptr_val
= roce_read(hr_dev
,
2687 ROCEE_SDB_SEND_PTR_REG
);
2688 if (!time_before(jiffies
, end
)) {
2689 dev_err(dev
, "destroy qp(0x%lx) timeout!!!",
2693 } while ((short)(roce_get_field(sdbsendptr_val
,
2694 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M
,
2695 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S
) -
2696 roce_get_field(sdbisusepr_val
,
2697 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M
,
2698 ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S
)
2701 /* Get list pointer */
2702 sdbinvcnt
= roce_read(hr_dev
, ROCEE_SDB_INV_CNT_REG
);
2704 /* Query db's list status, until hw reversal */
2706 sdbinvcnt_val
= roce_read(hr_dev
,
2707 ROCEE_SDB_INV_CNT_REG
);
2708 if (!time_before(jiffies
, end
)) {
2709 dev_err(dev
, "destroy qp(0x%lx) timeout!!!",
2711 dev_err(dev
, "SdbInvCnt = 0x%x\n",
2715 } while ((short)(roce_get_field(sdbinvcnt_val
,
2716 ROCEE_SDB_INV_CNT_SDB_INV_CNT_M
,
2717 ROCEE_SDB_INV_CNT_SDB_INV_CNT_S
) -
2718 (sdbinvcnt
+ SDB_INV_CNT_OFFSET
)) < 0);
2720 /* Modify qp to reset before destroying qp */
2721 if (hns_roce_v1_qp_modify(hr_dev
, NULL
,
2723 (enum ib_qp_state
)hr_qp
->state
),
2724 HNS_ROCE_QP_STATE_RST
, NULL
, hr_qp
))
2725 dev_err(dev
, "modify QP %06lx to RESET failed.\n",
2730 send_cq
= to_hr_cq(hr_qp
->ibqp
.send_cq
);
2731 recv_cq
= to_hr_cq(hr_qp
->ibqp
.recv_cq
);
2733 hns_roce_lock_cqs(send_cq
, recv_cq
);
2736 __hns_roce_v1_cq_clean(recv_cq
, hr_qp
->qpn
, hr_qp
->ibqp
.srq
?
2737 to_hr_srq(hr_qp
->ibqp
.srq
) : NULL
);
2738 if (send_cq
!= recv_cq
)
2739 __hns_roce_v1_cq_clean(send_cq
, hr_qp
->qpn
, NULL
);
2742 hns_roce_qp_remove(hr_dev
, hr_qp
);
2744 hns_roce_unlock_cqs(send_cq
, recv_cq
);
2746 hns_roce_qp_free(hr_dev
, hr_qp
);
2748 /* Not special_QP, free their QPN */
2749 if ((hr_qp
->ibqp
.qp_type
== IB_QPT_RC
) ||
2750 (hr_qp
->ibqp
.qp_type
== IB_QPT_UC
) ||
2751 (hr_qp
->ibqp
.qp_type
== IB_QPT_UD
))
2752 hns_roce_release_range_qp(hr_dev
, hr_qp
->qpn
, 1);
2754 hns_roce_mtt_cleanup(hr_dev
, &hr_qp
->mtt
);
2757 ib_umem_release(hr_qp
->umem
);
2759 kfree(hr_qp
->sq
.wrid
);
2760 kfree(hr_qp
->rq
.wrid
);
2761 hns_roce_buf_free(hr_dev
, hr_qp
->buff_size
, &hr_qp
->hr_buf
);
2765 int hns_roce_v1_destroy_qp(struct ib_qp
*ibqp
)
2767 struct hns_roce_dev
*hr_dev
= to_hr_dev(ibqp
->device
);
2768 struct hns_roce_qp
*hr_qp
= to_hr_qp(ibqp
);
2770 hns_roce_v1_destroy_qp_common(hr_dev
, hr_qp
, !!ibqp
->pd
->uobject
);
2772 if (hr_qp
->ibqp
.qp_type
== IB_QPT_GSI
)
2773 kfree(hr_to_hr_sqp(hr_qp
));
2780 struct hns_roce_v1_priv hr_v1_priv
;
2782 struct hns_roce_hw hns_roce_hw_v1
= {
2783 .reset
= hns_roce_v1_reset
,
2784 .hw_profile
= hns_roce_v1_profile
,
2785 .hw_init
= hns_roce_v1_init
,
2786 .hw_exit
= hns_roce_v1_exit
,
2787 .set_gid
= hns_roce_v1_set_gid
,
2788 .set_mac
= hns_roce_v1_set_mac
,
2789 .set_mtu
= hns_roce_v1_set_mtu
,
2790 .write_mtpt
= hns_roce_v1_write_mtpt
,
2791 .write_cqc
= hns_roce_v1_write_cqc
,
2792 .modify_qp
= hns_roce_v1_modify_qp
,
2793 .query_qp
= hns_roce_v1_query_qp
,
2794 .destroy_qp
= hns_roce_v1_destroy_qp
,
2795 .post_send
= hns_roce_v1_post_send
,
2796 .post_recv
= hns_roce_v1_post_recv
,
2797 .req_notify_cq
= hns_roce_v1_req_notify_cq
,
2798 .poll_cq
= hns_roce_v1_poll_cq
,
2799 .priv
= &hr_v1_priv
,