2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/gfp.h>
35 #include <linux/export.h>
36 #include <linux/mlx5/cmd.h>
37 #include <linux/mlx5/qp.h>
38 #include <linux/mlx5/driver.h>
40 #include "mlx5_core.h"
42 static struct mlx5_core_rsc_common
*mlx5_get_rsc(struct mlx5_core_dev
*dev
,
45 struct mlx5_qp_table
*table
= &dev
->priv
.qp_table
;
46 struct mlx5_core_rsc_common
*common
;
48 spin_lock(&table
->lock
);
50 common
= radix_tree_lookup(&table
->tree
, rsn
);
52 atomic_inc(&common
->refcount
);
54 spin_unlock(&table
->lock
);
57 mlx5_core_warn(dev
, "Async event for bogus resource 0x%x\n",
64 void mlx5_core_put_rsc(struct mlx5_core_rsc_common
*common
)
66 if (atomic_dec_and_test(&common
->refcount
))
67 complete(&common
->free
);
70 void mlx5_rsc_event(struct mlx5_core_dev
*dev
, u32 rsn
, int event_type
)
72 struct mlx5_core_rsc_common
*common
= mlx5_get_rsc(dev
, rsn
);
73 struct mlx5_core_qp
*qp
;
78 switch (common
->res
) {
80 qp
= (struct mlx5_core_qp
*)common
;
81 qp
->event(qp
, event_type
);
85 mlx5_core_warn(dev
, "invalid resource type for 0x%x\n", rsn
);
88 mlx5_core_put_rsc(common
);
91 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
92 void mlx5_eq_pagefault(struct mlx5_core_dev
*dev
, struct mlx5_eqe
*eqe
)
94 struct mlx5_eqe_page_fault
*pf_eqe
= &eqe
->data
.page_fault
;
95 int qpn
= be32_to_cpu(pf_eqe
->flags_qpn
) & MLX5_QPN_MASK
;
96 struct mlx5_core_rsc_common
*common
= mlx5_get_rsc(dev
, qpn
);
97 struct mlx5_core_qp
*qp
=
98 container_of(common
, struct mlx5_core_qp
, common
);
99 struct mlx5_pagefault pfault
;
102 mlx5_core_warn(dev
, "ODP event for non-existent QP %06x\n",
107 pfault
.event_subtype
= eqe
->sub_type
;
108 pfault
.flags
= (be32_to_cpu(pf_eqe
->flags_qpn
) >> MLX5_QPN_BITS
) &
109 (MLX5_PFAULT_REQUESTOR
| MLX5_PFAULT_WRITE
| MLX5_PFAULT_RDMA
);
110 pfault
.bytes_committed
= be32_to_cpu(
111 pf_eqe
->bytes_committed
);
114 "PAGE_FAULT: subtype: 0x%02x, flags: 0x%02x,\n",
115 eqe
->sub_type
, pfault
.flags
);
117 switch (eqe
->sub_type
) {
118 case MLX5_PFAULT_SUBTYPE_RDMA
:
119 /* RDMA based event */
121 be32_to_cpu(pf_eqe
->rdma
.r_key
);
122 pfault
.rdma
.packet_size
=
123 be16_to_cpu(pf_eqe
->rdma
.packet_length
);
124 pfault
.rdma
.rdma_op_len
=
125 be32_to_cpu(pf_eqe
->rdma
.rdma_op_len
);
126 pfault
.rdma
.rdma_va
=
127 be64_to_cpu(pf_eqe
->rdma
.rdma_va
);
129 "PAGE_FAULT: qpn: 0x%06x, r_key: 0x%08x,\n",
130 qpn
, pfault
.rdma
.r_key
);
132 "PAGE_FAULT: rdma_op_len: 0x%08x,\n",
133 pfault
.rdma
.rdma_op_len
);
135 "PAGE_FAULT: rdma_va: 0x%016llx,\n",
136 pfault
.rdma
.rdma_va
);
138 "PAGE_FAULT: bytes_committed: 0x%06x\n",
139 pfault
.bytes_committed
);
142 case MLX5_PFAULT_SUBTYPE_WQE
:
143 /* WQE based event */
144 pfault
.wqe
.wqe_index
=
145 be16_to_cpu(pf_eqe
->wqe
.wqe_index
);
146 pfault
.wqe
.packet_size
=
147 be16_to_cpu(pf_eqe
->wqe
.packet_length
);
149 "PAGE_FAULT: qpn: 0x%06x, wqe_index: 0x%04x,\n",
150 qpn
, pfault
.wqe
.wqe_index
);
152 "PAGE_FAULT: bytes_committed: 0x%06x\n",
153 pfault
.bytes_committed
);
158 "Unsupported page fault event sub-type: 0x%02hhx, QP %06x\n",
160 /* Unsupported page faults should still be resolved by the
165 if (qp
->pfault_handler
) {
166 qp
->pfault_handler(qp
, &pfault
);
169 "ODP event for QP %08x, without a fault handler in QP\n",
171 /* Page fault will remain unresolved. QP will hang until it is
176 mlx5_core_put_rsc(common
);
180 int mlx5_core_create_qp(struct mlx5_core_dev
*dev
,
181 struct mlx5_core_qp
*qp
,
182 struct mlx5_create_qp_mbox_in
*in
,
185 struct mlx5_qp_table
*table
= &dev
->priv
.qp_table
;
186 struct mlx5_create_qp_mbox_out out
;
187 struct mlx5_destroy_qp_mbox_in din
;
188 struct mlx5_destroy_qp_mbox_out dout
;
191 memset(&out
, 0, sizeof(out
));
192 in
->hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_CREATE_QP
);
194 err
= mlx5_cmd_exec(dev
, in
, inlen
, &out
, sizeof(out
));
196 mlx5_core_warn(dev
, "ret %d\n", err
);
200 if (out
.hdr
.status
) {
201 mlx5_core_warn(dev
, "current num of QPs 0x%x\n",
202 atomic_read(&dev
->num_qps
));
203 return mlx5_cmd_status_to_err(&out
.hdr
);
206 qp
->qpn
= be32_to_cpu(out
.qpn
) & 0xffffff;
207 mlx5_core_dbg(dev
, "qpn = 0x%x\n", qp
->qpn
);
209 qp
->common
.res
= MLX5_RES_QP
;
210 spin_lock_irq(&table
->lock
);
211 err
= radix_tree_insert(&table
->tree
, qp
->qpn
, qp
);
212 spin_unlock_irq(&table
->lock
);
214 mlx5_core_warn(dev
, "err %d\n", err
);
218 err
= mlx5_debug_qp_add(dev
, qp
);
220 mlx5_core_dbg(dev
, "failed adding QP 0x%x to debug file system\n",
223 qp
->pid
= current
->pid
;
224 atomic_set(&qp
->common
.refcount
, 1);
225 atomic_inc(&dev
->num_qps
);
226 init_completion(&qp
->common
.free
);
231 memset(&din
, 0, sizeof(din
));
232 memset(&dout
, 0, sizeof(dout
));
233 din
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_DESTROY_QP
);
234 din
.qpn
= cpu_to_be32(qp
->qpn
);
235 mlx5_cmd_exec(dev
, &din
, sizeof(din
), &out
, sizeof(dout
));
239 EXPORT_SYMBOL_GPL(mlx5_core_create_qp
);
241 int mlx5_core_destroy_qp(struct mlx5_core_dev
*dev
,
242 struct mlx5_core_qp
*qp
)
244 struct mlx5_destroy_qp_mbox_in in
;
245 struct mlx5_destroy_qp_mbox_out out
;
246 struct mlx5_qp_table
*table
= &dev
->priv
.qp_table
;
250 mlx5_debug_qp_remove(dev
, qp
);
252 spin_lock_irqsave(&table
->lock
, flags
);
253 radix_tree_delete(&table
->tree
, qp
->qpn
);
254 spin_unlock_irqrestore(&table
->lock
, flags
);
256 mlx5_core_put_rsc((struct mlx5_core_rsc_common
*)qp
);
257 wait_for_completion(&qp
->common
.free
);
259 memset(&in
, 0, sizeof(in
));
260 memset(&out
, 0, sizeof(out
));
261 in
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_DESTROY_QP
);
262 in
.qpn
= cpu_to_be32(qp
->qpn
);
263 err
= mlx5_cmd_exec(dev
, &in
, sizeof(in
), &out
, sizeof(out
));
268 return mlx5_cmd_status_to_err(&out
.hdr
);
270 atomic_dec(&dev
->num_qps
);
273 EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp
);
275 int mlx5_core_qp_modify(struct mlx5_core_dev
*dev
, enum mlx5_qp_state cur_state
,
276 enum mlx5_qp_state new_state
,
277 struct mlx5_modify_qp_mbox_in
*in
, int sqd_event
,
278 struct mlx5_core_qp
*qp
)
280 static const u16 optab
[MLX5_QP_NUM_STATE
][MLX5_QP_NUM_STATE
] = {
281 [MLX5_QP_STATE_RST
] = {
282 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
283 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
284 [MLX5_QP_STATE_INIT
] = MLX5_CMD_OP_RST2INIT_QP
,
286 [MLX5_QP_STATE_INIT
] = {
287 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
288 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
289 [MLX5_QP_STATE_INIT
] = MLX5_CMD_OP_INIT2INIT_QP
,
290 [MLX5_QP_STATE_RTR
] = MLX5_CMD_OP_INIT2RTR_QP
,
292 [MLX5_QP_STATE_RTR
] = {
293 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
294 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
295 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_RTR2RTS_QP
,
297 [MLX5_QP_STATE_RTS
] = {
298 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
299 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
300 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_RTS2RTS_QP
,
302 [MLX5_QP_STATE_SQD
] = {
303 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
304 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
306 [MLX5_QP_STATE_SQER
] = {
307 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
308 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
309 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_SQERR2RTS_QP
,
311 [MLX5_QP_STATE_ERR
] = {
312 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
313 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
317 struct mlx5_modify_qp_mbox_out out
;
321 if (cur_state
>= MLX5_QP_NUM_STATE
|| new_state
>= MLX5_QP_NUM_STATE
||
322 !optab
[cur_state
][new_state
])
325 memset(&out
, 0, sizeof(out
));
326 op
= optab
[cur_state
][new_state
];
327 in
->hdr
.opcode
= cpu_to_be16(op
);
328 in
->qpn
= cpu_to_be32(qp
->qpn
);
329 err
= mlx5_cmd_exec(dev
, in
, sizeof(*in
), &out
, sizeof(out
));
333 return mlx5_cmd_status_to_err(&out
.hdr
);
335 EXPORT_SYMBOL_GPL(mlx5_core_qp_modify
);
337 void mlx5_init_qp_table(struct mlx5_core_dev
*dev
)
339 struct mlx5_qp_table
*table
= &dev
->priv
.qp_table
;
341 spin_lock_init(&table
->lock
);
342 INIT_RADIX_TREE(&table
->tree
, GFP_ATOMIC
);
343 mlx5_qp_debugfs_init(dev
);
346 void mlx5_cleanup_qp_table(struct mlx5_core_dev
*dev
)
348 mlx5_qp_debugfs_cleanup(dev
);
351 int mlx5_core_qp_query(struct mlx5_core_dev
*dev
, struct mlx5_core_qp
*qp
,
352 struct mlx5_query_qp_mbox_out
*out
, int outlen
)
354 struct mlx5_query_qp_mbox_in in
;
357 memset(&in
, 0, sizeof(in
));
358 memset(out
, 0, outlen
);
359 in
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_QUERY_QP
);
360 in
.qpn
= cpu_to_be32(qp
->qpn
);
361 err
= mlx5_cmd_exec(dev
, &in
, sizeof(in
), out
, outlen
);
366 return mlx5_cmd_status_to_err(&out
->hdr
);
370 EXPORT_SYMBOL_GPL(mlx5_core_qp_query
);
372 int mlx5_core_xrcd_alloc(struct mlx5_core_dev
*dev
, u32
*xrcdn
)
374 struct mlx5_alloc_xrcd_mbox_in in
;
375 struct mlx5_alloc_xrcd_mbox_out out
;
378 memset(&in
, 0, sizeof(in
));
379 memset(&out
, 0, sizeof(out
));
380 in
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_ALLOC_XRCD
);
381 err
= mlx5_cmd_exec(dev
, &in
, sizeof(in
), &out
, sizeof(out
));
386 err
= mlx5_cmd_status_to_err(&out
.hdr
);
388 *xrcdn
= be32_to_cpu(out
.xrcdn
);
392 EXPORT_SYMBOL_GPL(mlx5_core_xrcd_alloc
);
394 int mlx5_core_xrcd_dealloc(struct mlx5_core_dev
*dev
, u32 xrcdn
)
396 struct mlx5_dealloc_xrcd_mbox_in in
;
397 struct mlx5_dealloc_xrcd_mbox_out out
;
400 memset(&in
, 0, sizeof(in
));
401 memset(&out
, 0, sizeof(out
));
402 in
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_DEALLOC_XRCD
);
403 in
.xrcdn
= cpu_to_be32(xrcdn
);
404 err
= mlx5_cmd_exec(dev
, &in
, sizeof(in
), &out
, sizeof(out
));
409 err
= mlx5_cmd_status_to_err(&out
.hdr
);
413 EXPORT_SYMBOL_GPL(mlx5_core_xrcd_dealloc
);
415 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
416 int mlx5_core_page_fault_resume(struct mlx5_core_dev
*dev
, u32 qpn
,
419 struct mlx5_page_fault_resume_mbox_in in
;
420 struct mlx5_page_fault_resume_mbox_out out
;
423 memset(&in
, 0, sizeof(in
));
424 memset(&out
, 0, sizeof(out
));
425 in
.hdr
.opcode
= cpu_to_be16(MLX5_CMD_OP_PAGE_FAULT_RESUME
);
427 flags
&= (MLX5_PAGE_FAULT_RESUME_REQUESTOR
|
428 MLX5_PAGE_FAULT_RESUME_WRITE
|
429 MLX5_PAGE_FAULT_RESUME_RDMA
);
430 flags
|= (error
? MLX5_PAGE_FAULT_RESUME_ERROR
: 0);
431 in
.flags_qpn
= cpu_to_be32((qpn
& MLX5_QPN_MASK
) |
432 (flags
<< MLX5_QPN_BITS
));
433 err
= mlx5_cmd_exec(dev
, &in
, sizeof(in
), &out
, sizeof(out
));
438 err
= mlx5_cmd_status_to_err(&out
.hdr
);
442 EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume
);