2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/etherdevice.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/mlx5_ifc.h>
36 #include <linux/mlx5/vport.h>
37 #include <linux/mlx5/fs.h>
38 #include "mlx5_core.h"
46 struct mlx5_flow_rule
*
47 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch
*esw
,
48 struct mlx5_flow_spec
*spec
,
49 u32 action
, u32 src_vport
, u32 dst_vport
)
51 struct mlx5_flow_destination dest
= { 0 };
52 struct mlx5_fc
*counter
= NULL
;
53 struct mlx5_flow_rule
*rule
;
56 if (esw
->mode
!= SRIOV_OFFLOADS
)
57 return ERR_PTR(-EOPNOTSUPP
);
59 if (action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
60 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
61 dest
.vport_num
= dst_vport
;
62 action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
63 } else if (action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
64 counter
= mlx5_fc_create(esw
->dev
, true);
66 return ERR_CAST(counter
);
67 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
68 dest
.counter
= counter
;
71 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
72 MLX5_SET(fte_match_set_misc
, misc
, source_port
, src_vport
);
74 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
75 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
77 spec
->match_criteria_enable
= MLX5_MATCH_OUTER_HEADERS
|
78 MLX5_MATCH_MISC_PARAMETERS
;
80 rule
= mlx5_add_flow_rule((struct mlx5_flow_table
*)esw
->fdb_table
.fdb
,
81 spec
, action
, 0, &dest
);
84 mlx5_fc_destroy(esw
->dev
, counter
);
89 static struct mlx5_flow_rule
*
90 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch
*esw
, int vport
, u32 sqn
)
92 struct mlx5_flow_destination dest
;
93 struct mlx5_flow_rule
*flow_rule
;
94 struct mlx5_flow_spec
*spec
;
97 spec
= mlx5_vzalloc(sizeof(*spec
));
99 esw_warn(esw
->dev
, "FDB: Failed to alloc match parameters\n");
100 flow_rule
= ERR_PTR(-ENOMEM
);
104 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
105 MLX5_SET(fte_match_set_misc
, misc
, source_sqn
, sqn
);
106 MLX5_SET(fte_match_set_misc
, misc
, source_port
, 0x0); /* source vport is 0 */
108 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
109 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_sqn
);
110 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
112 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
113 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
114 dest
.vport_num
= vport
;
116 flow_rule
= mlx5_add_flow_rule(esw
->fdb_table
.offloads
.fdb
, spec
,
117 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
,
119 if (IS_ERR(flow_rule
))
120 esw_warn(esw
->dev
, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule
));
126 void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch
*esw
,
127 struct mlx5_eswitch_rep
*rep
)
129 struct mlx5_esw_sq
*esw_sq
, *tmp
;
131 if (esw
->mode
!= SRIOV_OFFLOADS
)
134 list_for_each_entry_safe(esw_sq
, tmp
, &rep
->vport_sqs_list
, list
) {
135 mlx5_del_flow_rule(esw_sq
->send_to_vport_rule
);
136 list_del(&esw_sq
->list
);
141 int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch
*esw
,
142 struct mlx5_eswitch_rep
*rep
,
143 u16
*sqns_array
, int sqns_num
)
145 struct mlx5_flow_rule
*flow_rule
;
146 struct mlx5_esw_sq
*esw_sq
;
151 if (esw
->mode
!= SRIOV_OFFLOADS
)
154 vport
= rep
->vport
== 0 ?
155 FDB_UPLINK_VPORT
: rep
->vport
;
157 for (i
= 0; i
< sqns_num
; i
++) {
158 esw_sq
= kzalloc(sizeof(*esw_sq
), GFP_KERNEL
);
164 /* Add re-inject rule to the PF/representor sqs */
165 flow_rule
= mlx5_eswitch_add_send_to_vport_rule(esw
,
168 if (IS_ERR(flow_rule
)) {
169 err
= PTR_ERR(flow_rule
);
173 esw_sq
->send_to_vport_rule
= flow_rule
;
174 list_add(&esw_sq
->list
, &rep
->vport_sqs_list
);
179 mlx5_eswitch_sqs2vport_stop(esw
, rep
);
183 static int esw_add_fdb_miss_rule(struct mlx5_eswitch
*esw
)
185 struct mlx5_flow_destination dest
;
186 struct mlx5_flow_rule
*flow_rule
= NULL
;
187 struct mlx5_flow_spec
*spec
;
190 spec
= mlx5_vzalloc(sizeof(*spec
));
192 esw_warn(esw
->dev
, "FDB: Failed to alloc match parameters\n");
197 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
200 flow_rule
= mlx5_add_flow_rule(esw
->fdb_table
.offloads
.fdb
, spec
,
201 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
,
203 if (IS_ERR(flow_rule
)) {
204 err
= PTR_ERR(flow_rule
);
205 esw_warn(esw
->dev
, "FDB: Failed to add miss flow rule err %d\n", err
);
209 esw
->fdb_table
.offloads
.miss_rule
= flow_rule
;
215 #define MAX_PF_SQ 256
216 #define ESW_OFFLOADS_NUM_ENTRIES (1 << 13) /* 8K */
217 #define ESW_OFFLOADS_NUM_GROUPS 4
219 static int esw_create_offloads_fdb_table(struct mlx5_eswitch
*esw
, int nvports
)
221 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
222 struct mlx5_core_dev
*dev
= esw
->dev
;
223 struct mlx5_flow_namespace
*root_ns
;
224 struct mlx5_flow_table
*fdb
= NULL
;
225 struct mlx5_flow_group
*g
;
227 void *match_criteria
;
228 int table_size
, ix
, err
= 0;
230 flow_group_in
= mlx5_vzalloc(inlen
);
234 root_ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_FDB
);
236 esw_warn(dev
, "Failed to get FDB flow namespace\n");
240 esw_debug(dev
, "Create offloads FDB table, log_max_size(%d)\n",
241 MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, log_max_ft_size
));
243 fdb
= mlx5_create_auto_grouped_flow_table(root_ns
, FDB_FAST_PATH
,
244 ESW_OFFLOADS_NUM_ENTRIES
,
245 ESW_OFFLOADS_NUM_GROUPS
, 0);
248 esw_warn(dev
, "Failed to create Fast path FDB Table err %d\n", err
);
251 esw
->fdb_table
.fdb
= fdb
;
253 table_size
= nvports
+ MAX_PF_SQ
+ 1;
254 fdb
= mlx5_create_flow_table(root_ns
, FDB_SLOW_PATH
, table_size
, 0);
257 esw_warn(dev
, "Failed to create slow path FDB Table err %d\n", err
);
260 esw
->fdb_table
.offloads
.fdb
= fdb
;
262 /* create send-to-vport group */
263 memset(flow_group_in
, 0, inlen
);
264 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
265 MLX5_MATCH_MISC_PARAMETERS
);
267 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
269 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_sqn
);
270 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_port
);
272 ix
= nvports
+ MAX_PF_SQ
;
273 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
274 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, ix
- 1);
276 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
279 esw_warn(dev
, "Failed to create send-to-vport flow group err(%d)\n", err
);
282 esw
->fdb_table
.offloads
.send_to_vport_grp
= g
;
284 /* create miss group */
285 memset(flow_group_in
, 0, inlen
);
286 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
, 0);
288 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, ix
);
289 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, ix
+ 1);
291 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
294 esw_warn(dev
, "Failed to create miss flow group err(%d)\n", err
);
297 esw
->fdb_table
.offloads
.miss_grp
= g
;
299 err
= esw_add_fdb_miss_rule(esw
);
306 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
308 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
310 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.fdb
);
312 mlx5_destroy_flow_table(esw
->fdb_table
.fdb
);
315 kvfree(flow_group_in
);
319 static void esw_destroy_offloads_fdb_table(struct mlx5_eswitch
*esw
)
321 if (!esw
->fdb_table
.fdb
)
324 esw_debug(esw
->dev
, "Destroy offloads FDB Table\n");
325 mlx5_del_flow_rule(esw
->fdb_table
.offloads
.miss_rule
);
326 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
327 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
329 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.fdb
);
330 mlx5_destroy_flow_table(esw
->fdb_table
.fdb
);
333 static int esw_create_offloads_table(struct mlx5_eswitch
*esw
)
335 struct mlx5_flow_namespace
*ns
;
336 struct mlx5_flow_table
*ft_offloads
;
337 struct mlx5_core_dev
*dev
= esw
->dev
;
340 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
342 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
346 ft_offloads
= mlx5_create_flow_table(ns
, 0, dev
->priv
.sriov
.num_vfs
+ 2, 0);
347 if (IS_ERR(ft_offloads
)) {
348 err
= PTR_ERR(ft_offloads
);
349 esw_warn(esw
->dev
, "Failed to create offloads table, err %d\n", err
);
353 esw
->offloads
.ft_offloads
= ft_offloads
;
357 static void esw_destroy_offloads_table(struct mlx5_eswitch
*esw
)
359 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
361 mlx5_destroy_flow_table(offloads
->ft_offloads
);
364 static int esw_create_vport_rx_group(struct mlx5_eswitch
*esw
)
366 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
367 struct mlx5_flow_group
*g
;
368 struct mlx5_priv
*priv
= &esw
->dev
->priv
;
370 void *match_criteria
, *misc
;
372 int nvports
= priv
->sriov
.num_vfs
+ 2;
374 flow_group_in
= mlx5_vzalloc(inlen
);
378 /* create vport rx group */
379 memset(flow_group_in
, 0, inlen
);
380 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
381 MLX5_MATCH_MISC_PARAMETERS
);
383 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
384 misc
= MLX5_ADDR_OF(fte_match_param
, match_criteria
, misc_parameters
);
385 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
387 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
388 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, nvports
- 1);
390 g
= mlx5_create_flow_group(esw
->offloads
.ft_offloads
, flow_group_in
);
394 mlx5_core_warn(esw
->dev
, "Failed to create vport rx group err %d\n", err
);
398 esw
->offloads
.vport_rx_group
= g
;
400 kfree(flow_group_in
);
404 static void esw_destroy_vport_rx_group(struct mlx5_eswitch
*esw
)
406 mlx5_destroy_flow_group(esw
->offloads
.vport_rx_group
);
409 struct mlx5_flow_rule
*
410 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch
*esw
, int vport
, u32 tirn
)
412 struct mlx5_flow_destination dest
;
413 struct mlx5_flow_rule
*flow_rule
;
414 struct mlx5_flow_spec
*spec
;
417 spec
= mlx5_vzalloc(sizeof(*spec
));
419 esw_warn(esw
->dev
, "Failed to alloc match parameters\n");
420 flow_rule
= ERR_PTR(-ENOMEM
);
424 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
425 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
427 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
428 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
430 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
431 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
434 flow_rule
= mlx5_add_flow_rule(esw
->offloads
.ft_offloads
, spec
,
435 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
,
437 if (IS_ERR(flow_rule
)) {
438 esw_warn(esw
->dev
, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule
));
447 static int esw_offloads_start(struct mlx5_eswitch
*esw
)
449 int err
, num_vfs
= esw
->dev
->priv
.sriov
.num_vfs
;
451 if (esw
->mode
!= SRIOV_LEGACY
) {
452 esw_warn(esw
->dev
, "Can't set offloads mode, SRIOV legacy not enabled\n");
456 mlx5_eswitch_disable_sriov(esw
);
457 err
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_OFFLOADS
);
459 esw_warn(esw
->dev
, "Failed set eswitch to offloads, err %d\n", err
);
463 int esw_offloads_init(struct mlx5_eswitch
*esw
, int nvports
)
465 struct mlx5_eswitch_rep
*rep
;
469 err
= esw_create_offloads_fdb_table(esw
, nvports
);
473 err
= esw_create_offloads_table(esw
);
477 err
= esw_create_vport_rx_group(esw
);
481 for (vport
= 0; vport
< nvports
; vport
++) {
482 rep
= &esw
->offloads
.vport_reps
[vport
];
486 err
= rep
->load(esw
, rep
);
493 for (vport
--; vport
>= 0; vport
--) {
494 rep
= &esw
->offloads
.vport_reps
[vport
];
497 rep
->unload(esw
, rep
);
499 esw_destroy_vport_rx_group(esw
);
502 esw_destroy_offloads_table(esw
);
505 esw_destroy_offloads_fdb_table(esw
);
509 static int esw_offloads_stop(struct mlx5_eswitch
*esw
)
511 int err
, num_vfs
= esw
->dev
->priv
.sriov
.num_vfs
;
513 mlx5_eswitch_disable_sriov(esw
);
514 err
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_LEGACY
);
516 esw_warn(esw
->dev
, "Failed set eswitch legacy mode. err %d\n", err
);
521 void esw_offloads_cleanup(struct mlx5_eswitch
*esw
, int nvports
)
523 struct mlx5_eswitch_rep
*rep
;
526 for (vport
= 0; vport
< nvports
; vport
++) {
527 rep
= &esw
->offloads
.vport_reps
[vport
];
530 rep
->unload(esw
, rep
);
533 esw_destroy_vport_rx_group(esw
);
534 esw_destroy_offloads_table(esw
);
535 esw_destroy_offloads_fdb_table(esw
);
538 static int esw_mode_from_devlink(u16 mode
, u16
*mlx5_mode
)
541 case DEVLINK_ESWITCH_MODE_LEGACY
:
542 *mlx5_mode
= SRIOV_LEGACY
;
544 case DEVLINK_ESWITCH_MODE_SWITCHDEV
:
545 *mlx5_mode
= SRIOV_OFFLOADS
;
554 static int esw_mode_to_devlink(u16 mlx5_mode
, u16
*mode
)
558 *mode
= DEVLINK_ESWITCH_MODE_LEGACY
;
561 *mode
= DEVLINK_ESWITCH_MODE_SWITCHDEV
;
570 int mlx5_devlink_eswitch_mode_set(struct devlink
*devlink
, u16 mode
)
572 struct mlx5_core_dev
*dev
;
573 u16 cur_mlx5_mode
, mlx5_mode
= 0;
575 dev
= devlink_priv(devlink
);
577 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
580 cur_mlx5_mode
= dev
->priv
.eswitch
->mode
;
582 if (cur_mlx5_mode
== SRIOV_NONE
)
585 if (esw_mode_from_devlink(mode
, &mlx5_mode
))
588 if (cur_mlx5_mode
== mlx5_mode
)
591 if (mode
== DEVLINK_ESWITCH_MODE_SWITCHDEV
)
592 return esw_offloads_start(dev
->priv
.eswitch
);
593 else if (mode
== DEVLINK_ESWITCH_MODE_LEGACY
)
594 return esw_offloads_stop(dev
->priv
.eswitch
);
599 int mlx5_devlink_eswitch_mode_get(struct devlink
*devlink
, u16
*mode
)
601 struct mlx5_core_dev
*dev
;
603 dev
= devlink_priv(devlink
);
605 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
608 if (dev
->priv
.eswitch
->mode
== SRIOV_NONE
)
611 return esw_mode_to_devlink(dev
->priv
.eswitch
->mode
, mode
);
614 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch
*esw
,
615 struct mlx5_eswitch_rep
*rep
)
617 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
619 memcpy(&offloads
->vport_reps
[rep
->vport
], rep
,
620 sizeof(struct mlx5_eswitch_rep
));
622 INIT_LIST_HEAD(&offloads
->vport_reps
[rep
->vport
].vport_sqs_list
);
623 offloads
->vport_reps
[rep
->vport
].valid
= true;
626 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch
*esw
,
629 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
630 struct mlx5_eswitch_rep
*rep
;
632 rep
= &offloads
->vport_reps
[vport
];
634 if (esw
->mode
== SRIOV_OFFLOADS
&& esw
->vports
[vport
].enabled
)
635 rep
->unload(esw
, rep
);
637 offloads
->vport_reps
[vport
].valid
= false;