2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/mutex.h>
34 #include <linux/mlx5/driver.h>
36 #include "mlx5_core.h"
40 #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
41 sizeof(struct init_tree_node))
43 #define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
44 ...) {.type = FS_TYPE_PRIO,\
45 .min_ft_level = min_level_val,\
46 .num_levels = num_levels_val,\
47 .num_leaf_prios = num_prios_val,\
49 .children = (struct init_tree_node[]) {__VA_ARGS__},\
50 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
53 #define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
54 ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
57 #define ADD_NS(...) {.type = FS_TYPE_NAMESPACE,\
58 .children = (struct init_tree_node[]) {__VA_ARGS__},\
59 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
62 #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
65 #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
67 #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
68 .caps = (long[]) {__VA_ARGS__} }
70 #define FS_CHAINING_CAPS FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
71 FS_CAP(flow_table_properties_nic_receive.modify_root), \
72 FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
73 FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
75 #define LEFTOVERS_NUM_LEVELS 1
76 #define LEFTOVERS_NUM_PRIOS 1
78 #define BY_PASS_PRIO_NUM_LEVELS 1
79 #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
82 #define ETHTOOL_PRIO_NUM_LEVELS 1
83 #define ETHTOOL_NUM_PRIOS 11
84 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
85 /* Vlan, mac, ttc, aRFS */
86 #define KERNEL_NIC_PRIO_NUM_LEVELS 4
87 #define KERNEL_NIC_NUM_PRIOS 1
88 /* One more level for tc */
89 #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
91 #define ANCHOR_NUM_LEVELS 1
92 #define ANCHOR_NUM_PRIOS 1
93 #define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
95 #define OFFLOADS_MAX_FT 1
96 #define OFFLOADS_NUM_PRIOS 1
97 #define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + 1)
103 static struct init_tree_node
{
104 enum fs_node_type type
;
105 struct init_tree_node
*children
;
107 struct node_caps caps
;
113 .type
= FS_TYPE_NAMESPACE
,
115 .children
= (struct init_tree_node
[]) {
116 ADD_PRIO(0, BY_PASS_MIN_LEVEL
, 0,
118 ADD_NS(ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS
,
119 BY_PASS_PRIO_NUM_LEVELS
))),
120 ADD_PRIO(0, OFFLOADS_MIN_LEVEL
, 0, {},
121 ADD_NS(ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS
, OFFLOADS_MAX_FT
))),
122 ADD_PRIO(0, ETHTOOL_MIN_LEVEL
, 0,
124 ADD_NS(ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS
,
125 ETHTOOL_PRIO_NUM_LEVELS
))),
126 ADD_PRIO(0, KERNEL_MIN_LEVEL
, 0, {},
127 ADD_NS(ADD_MULTIPLE_PRIO(1, 1),
128 ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS
,
129 KERNEL_NIC_PRIO_NUM_LEVELS
))),
130 ADD_PRIO(0, BY_PASS_MIN_LEVEL
, 0,
132 ADD_NS(ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS
, LEFTOVERS_NUM_LEVELS
))),
133 ADD_PRIO(0, ANCHOR_MIN_LEVEL
, 0, {},
134 ADD_NS(ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS
, ANCHOR_NUM_LEVELS
))),
138 enum fs_i_mutex_lock_class
{
139 FS_MUTEX_GRANDPARENT
,
144 static void del_rule(struct fs_node
*node
);
145 static void del_flow_table(struct fs_node
*node
);
146 static void del_flow_group(struct fs_node
*node
);
147 static void del_fte(struct fs_node
*node
);
149 static void tree_init_node(struct fs_node
*node
,
150 unsigned int refcount
,
151 void (*remove_func
)(struct fs_node
*))
153 atomic_set(&node
->refcount
, refcount
);
154 INIT_LIST_HEAD(&node
->list
);
155 INIT_LIST_HEAD(&node
->children
);
156 mutex_init(&node
->lock
);
157 node
->remove_func
= remove_func
;
160 static void tree_add_node(struct fs_node
*node
, struct fs_node
*parent
)
163 atomic_inc(&parent
->refcount
);
164 node
->parent
= parent
;
166 /* Parent is the root */
170 node
->root
= parent
->root
;
173 static void tree_get_node(struct fs_node
*node
)
175 atomic_inc(&node
->refcount
);
178 static void nested_lock_ref_node(struct fs_node
*node
,
179 enum fs_i_mutex_lock_class
class)
182 mutex_lock_nested(&node
->lock
, class);
183 atomic_inc(&node
->refcount
);
187 static void lock_ref_node(struct fs_node
*node
)
190 mutex_lock(&node
->lock
);
191 atomic_inc(&node
->refcount
);
195 static void unlock_ref_node(struct fs_node
*node
)
198 atomic_dec(&node
->refcount
);
199 mutex_unlock(&node
->lock
);
203 static void tree_put_node(struct fs_node
*node
)
205 struct fs_node
*parent_node
= node
->parent
;
207 lock_ref_node(parent_node
);
208 if (atomic_dec_and_test(&node
->refcount
)) {
210 list_del_init(&node
->list
);
211 if (node
->remove_func
)
212 node
->remove_func(node
);
216 unlock_ref_node(parent_node
);
217 if (!node
&& parent_node
)
218 tree_put_node(parent_node
);
221 static int tree_remove_node(struct fs_node
*node
)
223 if (atomic_read(&node
->refcount
) > 1) {
224 atomic_dec(&node
->refcount
);
231 static struct fs_prio
*find_prio(struct mlx5_flow_namespace
*ns
,
234 struct fs_prio
*iter_prio
;
236 fs_for_each_prio(iter_prio
, ns
) {
237 if (iter_prio
->prio
== prio
)
244 static bool masked_memcmp(void *mask
, void *val1
, void *val2
, size_t size
)
248 for (i
= 0; i
< size
; i
++, mask
++, val1
++, val2
++)
249 if ((*((u8
*)val1
) & (*(u8
*)mask
)) !=
250 ((*(u8
*)val2
) & (*(u8
*)mask
)))
256 static bool compare_match_value(struct mlx5_flow_group_mask
*mask
,
257 void *fte_param1
, void *fte_param2
)
259 if (mask
->match_criteria_enable
&
260 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS
) {
261 void *fte_match1
= MLX5_ADDR_OF(fte_match_param
,
262 fte_param1
, outer_headers
);
263 void *fte_match2
= MLX5_ADDR_OF(fte_match_param
,
264 fte_param2
, outer_headers
);
265 void *fte_mask
= MLX5_ADDR_OF(fte_match_param
,
266 mask
->match_criteria
, outer_headers
);
268 if (!masked_memcmp(fte_mask
, fte_match1
, fte_match2
,
269 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4
)))
273 if (mask
->match_criteria_enable
&
274 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS
) {
275 void *fte_match1
= MLX5_ADDR_OF(fte_match_param
,
276 fte_param1
, misc_parameters
);
277 void *fte_match2
= MLX5_ADDR_OF(fte_match_param
,
278 fte_param2
, misc_parameters
);
279 void *fte_mask
= MLX5_ADDR_OF(fte_match_param
,
280 mask
->match_criteria
, misc_parameters
);
282 if (!masked_memcmp(fte_mask
, fte_match1
, fte_match2
,
283 MLX5_ST_SZ_BYTES(fte_match_set_misc
)))
287 if (mask
->match_criteria_enable
&
288 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS
) {
289 void *fte_match1
= MLX5_ADDR_OF(fte_match_param
,
290 fte_param1
, inner_headers
);
291 void *fte_match2
= MLX5_ADDR_OF(fte_match_param
,
292 fte_param2
, inner_headers
);
293 void *fte_mask
= MLX5_ADDR_OF(fte_match_param
,
294 mask
->match_criteria
, inner_headers
);
296 if (!masked_memcmp(fte_mask
, fte_match1
, fte_match2
,
297 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4
)))
303 static bool compare_match_criteria(u8 match_criteria_enable1
,
304 u8 match_criteria_enable2
,
305 void *mask1
, void *mask2
)
307 return match_criteria_enable1
== match_criteria_enable2
&&
308 !memcmp(mask1
, mask2
, MLX5_ST_SZ_BYTES(fte_match_param
));
311 static struct mlx5_flow_root_namespace
*find_root(struct fs_node
*node
)
313 struct fs_node
*root
;
314 struct mlx5_flow_namespace
*ns
;
318 if (WARN_ON(root
->type
!= FS_TYPE_NAMESPACE
)) {
319 pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
323 ns
= container_of(root
, struct mlx5_flow_namespace
, node
);
324 return container_of(ns
, struct mlx5_flow_root_namespace
, ns
);
327 static inline struct mlx5_core_dev
*get_dev(struct fs_node
*node
)
329 struct mlx5_flow_root_namespace
*root
= find_root(node
);
336 static void del_flow_table(struct fs_node
*node
)
338 struct mlx5_flow_table
*ft
;
339 struct mlx5_core_dev
*dev
;
340 struct fs_prio
*prio
;
343 fs_get_obj(ft
, node
);
344 dev
= get_dev(&ft
->node
);
346 err
= mlx5_cmd_destroy_flow_table(dev
, ft
);
348 pr_warn("flow steering can't destroy ft\n");
349 fs_get_obj(prio
, ft
->node
.parent
);
353 static void del_rule(struct fs_node
*node
)
355 struct mlx5_flow_rule
*rule
;
356 struct mlx5_flow_table
*ft
;
357 struct mlx5_flow_group
*fg
;
361 struct mlx5_core_dev
*dev
= get_dev(node
);
362 int match_len
= MLX5_ST_SZ_BYTES(fte_match_param
);
365 match_value
= mlx5_vzalloc(match_len
);
367 pr_warn("failed to allocate inbox\n");
371 fs_get_obj(rule
, node
);
372 fs_get_obj(fte
, rule
->node
.parent
);
373 fs_get_obj(fg
, fte
->node
.parent
);
374 memcpy(match_value
, fte
->val
, sizeof(fte
->val
));
375 fs_get_obj(ft
, fg
->node
.parent
);
376 list_del(&rule
->node
.list
);
377 if (rule
->sw_action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
378 mutex_lock(&rule
->dest_attr
.ft
->lock
);
379 list_del(&rule
->next_ft
);
380 mutex_unlock(&rule
->dest_attr
.ft
->lock
);
382 if ((fte
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) &&
384 modify_mask
= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
),
385 err
= mlx5_cmd_update_fte(dev
, ft
,
390 pr_warn("%s can't del rule fg id=%d fte_index=%d\n",
391 __func__
, fg
->id
, fte
->index
);
396 static void del_fte(struct fs_node
*node
)
398 struct mlx5_flow_table
*ft
;
399 struct mlx5_flow_group
*fg
;
400 struct mlx5_core_dev
*dev
;
404 fs_get_obj(fte
, node
);
405 fs_get_obj(fg
, fte
->node
.parent
);
406 fs_get_obj(ft
, fg
->node
.parent
);
408 dev
= get_dev(&ft
->node
);
409 err
= mlx5_cmd_delete_fte(dev
, ft
,
412 pr_warn("flow steering can't delete fte in index %d of flow group id %d\n",
419 static void del_flow_group(struct fs_node
*node
)
421 struct mlx5_flow_group
*fg
;
422 struct mlx5_flow_table
*ft
;
423 struct mlx5_core_dev
*dev
;
425 fs_get_obj(fg
, node
);
426 fs_get_obj(ft
, fg
->node
.parent
);
427 dev
= get_dev(&ft
->node
);
429 if (mlx5_cmd_destroy_flow_group(dev
, ft
, fg
->id
))
430 pr_warn("flow steering can't destroy fg %d of ft %d\n",
434 static struct fs_fte
*alloc_fte(u8 action
,
441 fte
= kzalloc(sizeof(*fte
), GFP_KERNEL
);
443 return ERR_PTR(-ENOMEM
);
445 memcpy(fte
->val
, match_value
, sizeof(fte
->val
));
446 fte
->node
.type
= FS_TYPE_FLOW_ENTRY
;
447 fte
->flow_tag
= flow_tag
;
449 fte
->action
= action
;
454 static struct mlx5_flow_group
*alloc_flow_group(u32
*create_fg_in
)
456 struct mlx5_flow_group
*fg
;
457 void *match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
458 create_fg_in
, match_criteria
);
459 u8 match_criteria_enable
= MLX5_GET(create_flow_group_in
,
461 match_criteria_enable
);
462 fg
= kzalloc(sizeof(*fg
), GFP_KERNEL
);
464 return ERR_PTR(-ENOMEM
);
466 fg
->mask
.match_criteria_enable
= match_criteria_enable
;
467 memcpy(&fg
->mask
.match_criteria
, match_criteria
,
468 sizeof(fg
->mask
.match_criteria
));
469 fg
->node
.type
= FS_TYPE_FLOW_GROUP
;
470 fg
->start_index
= MLX5_GET(create_flow_group_in
, create_fg_in
,
472 fg
->max_ftes
= MLX5_GET(create_flow_group_in
, create_fg_in
,
473 end_flow_index
) - fg
->start_index
+ 1;
477 static struct mlx5_flow_table
*alloc_flow_table(int level
, u16 vport
, int max_fte
,
478 enum fs_flow_table_type table_type
)
480 struct mlx5_flow_table
*ft
;
482 ft
= kzalloc(sizeof(*ft
), GFP_KERNEL
);
487 ft
->node
.type
= FS_TYPE_FLOW_TABLE
;
488 ft
->type
= table_type
;
490 ft
->max_fte
= max_fte
;
491 INIT_LIST_HEAD(&ft
->fwd_rules
);
492 mutex_init(&ft
->lock
);
497 /* If reverse is false, then we search for the first flow table in the
498 * root sub-tree from start(closest from right), else we search for the
499 * last flow table in the root sub-tree till start(closest from left).
501 static struct mlx5_flow_table
*find_closest_ft_recursive(struct fs_node
*root
,
502 struct list_head
*start
,
505 #define list_advance_entry(pos, reverse) \
506 ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
508 #define list_for_each_advance_continue(pos, head, reverse) \
509 for (pos = list_advance_entry(pos, reverse); \
510 &pos->list != (head); \
511 pos = list_advance_entry(pos, reverse))
513 struct fs_node
*iter
= list_entry(start
, struct fs_node
, list
);
514 struct mlx5_flow_table
*ft
= NULL
;
519 list_for_each_advance_continue(iter
, &root
->children
, reverse
) {
520 if (iter
->type
== FS_TYPE_FLOW_TABLE
) {
521 fs_get_obj(ft
, iter
);
524 ft
= find_closest_ft_recursive(iter
, &iter
->children
, reverse
);
532 /* If reverse if false then return the first flow table in next priority of
533 * prio in the tree, else return the last flow table in the previous priority
534 * of prio in the tree.
536 static struct mlx5_flow_table
*find_closest_ft(struct fs_prio
*prio
, bool reverse
)
538 struct mlx5_flow_table
*ft
= NULL
;
539 struct fs_node
*curr_node
;
540 struct fs_node
*parent
;
542 parent
= prio
->node
.parent
;
543 curr_node
= &prio
->node
;
544 while (!ft
&& parent
) {
545 ft
= find_closest_ft_recursive(parent
, &curr_node
->list
, reverse
);
547 parent
= curr_node
->parent
;
552 /* Assuming all the tree is locked by mutex chain lock */
553 static struct mlx5_flow_table
*find_next_chained_ft(struct fs_prio
*prio
)
555 return find_closest_ft(prio
, false);
558 /* Assuming all the tree is locked by mutex chain lock */
559 static struct mlx5_flow_table
*find_prev_chained_ft(struct fs_prio
*prio
)
561 return find_closest_ft(prio
, true);
564 static int connect_fts_in_prio(struct mlx5_core_dev
*dev
,
565 struct fs_prio
*prio
,
566 struct mlx5_flow_table
*ft
)
568 struct mlx5_flow_table
*iter
;
572 fs_for_each_ft(iter
, prio
) {
574 err
= mlx5_cmd_modify_flow_table(dev
,
578 mlx5_core_warn(dev
, "Failed to modify flow table %d\n",
580 /* The driver is out of sync with the FW */
589 /* Connect flow tables from previous priority of prio to ft */
590 static int connect_prev_fts(struct mlx5_core_dev
*dev
,
591 struct mlx5_flow_table
*ft
,
592 struct fs_prio
*prio
)
594 struct mlx5_flow_table
*prev_ft
;
596 prev_ft
= find_prev_chained_ft(prio
);
598 struct fs_prio
*prev_prio
;
600 fs_get_obj(prev_prio
, prev_ft
->node
.parent
);
601 return connect_fts_in_prio(dev
, prev_prio
, ft
);
606 static int update_root_ft_create(struct mlx5_flow_table
*ft
, struct fs_prio
609 struct mlx5_flow_root_namespace
*root
= find_root(&prio
->node
);
610 int min_level
= INT_MAX
;
614 min_level
= root
->root_ft
->level
;
616 if (ft
->level
>= min_level
)
619 err
= mlx5_cmd_update_root_ft(root
->dev
, ft
);
621 mlx5_core_warn(root
->dev
, "Update root flow table of id=%u failed\n",
629 int mlx5_modify_rule_destination(struct mlx5_flow_rule
*rule
,
630 struct mlx5_flow_destination
*dest
)
632 struct mlx5_flow_table
*ft
;
633 struct mlx5_flow_group
*fg
;
635 int modify_mask
= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
);
638 fs_get_obj(fte
, rule
->node
.parent
);
639 if (!(fte
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
))
641 lock_ref_node(&fte
->node
);
642 fs_get_obj(fg
, fte
->node
.parent
);
643 fs_get_obj(ft
, fg
->node
.parent
);
645 memcpy(&rule
->dest_attr
, dest
, sizeof(*dest
));
646 err
= mlx5_cmd_update_fte(get_dev(&ft
->node
),
650 unlock_ref_node(&fte
->node
);
655 /* Modify/set FWD rules that point on old_next_ft to point on new_next_ft */
656 static int connect_fwd_rules(struct mlx5_core_dev
*dev
,
657 struct mlx5_flow_table
*new_next_ft
,
658 struct mlx5_flow_table
*old_next_ft
)
660 struct mlx5_flow_destination dest
;
661 struct mlx5_flow_rule
*iter
;
664 /* new_next_ft and old_next_ft could be NULL only
665 * when we create/destroy the anchor flow table.
667 if (!new_next_ft
|| !old_next_ft
)
670 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
671 dest
.ft
= new_next_ft
;
673 mutex_lock(&old_next_ft
->lock
);
674 list_splice_init(&old_next_ft
->fwd_rules
, &new_next_ft
->fwd_rules
);
675 mutex_unlock(&old_next_ft
->lock
);
676 list_for_each_entry(iter
, &new_next_ft
->fwd_rules
, next_ft
) {
677 err
= mlx5_modify_rule_destination(iter
, &dest
);
679 pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
685 static int connect_flow_table(struct mlx5_core_dev
*dev
, struct mlx5_flow_table
*ft
,
686 struct fs_prio
*prio
)
688 struct mlx5_flow_table
*next_ft
;
691 /* Connect_prev_fts and update_root_ft_create are mutually exclusive */
693 if (list_empty(&prio
->node
.children
)) {
694 err
= connect_prev_fts(dev
, ft
, prio
);
698 next_ft
= find_next_chained_ft(prio
);
699 err
= connect_fwd_rules(dev
, ft
, next_ft
);
704 if (MLX5_CAP_FLOWTABLE(dev
,
705 flow_table_properties_nic_receive
.modify_root
))
706 err
= update_root_ft_create(ft
, prio
);
710 static void list_add_flow_table(struct mlx5_flow_table
*ft
,
711 struct fs_prio
*prio
)
713 struct list_head
*prev
= &prio
->node
.children
;
714 struct mlx5_flow_table
*iter
;
716 fs_for_each_ft(iter
, prio
) {
717 if (iter
->level
> ft
->level
)
719 prev
= &iter
->node
.list
;
721 list_add(&ft
->node
.list
, prev
);
724 static struct mlx5_flow_table
*__mlx5_create_flow_table(struct mlx5_flow_namespace
*ns
,
726 int max_fte
, u32 level
)
728 struct mlx5_flow_table
*next_ft
= NULL
;
729 struct mlx5_flow_table
*ft
;
732 struct mlx5_flow_root_namespace
*root
=
733 find_root(&ns
->node
);
734 struct fs_prio
*fs_prio
= NULL
;
737 pr_err("mlx5: flow steering failed to find root of namespace\n");
738 return ERR_PTR(-ENODEV
);
741 mutex_lock(&root
->chain_lock
);
742 fs_prio
= find_prio(ns
, prio
);
747 if (level
>= fs_prio
->num_levels
) {
751 /* The level is related to the
752 * priority level range.
754 level
+= fs_prio
->start_level
;
755 ft
= alloc_flow_table(level
,
757 roundup_pow_of_two(max_fte
),
764 tree_init_node(&ft
->node
, 1, del_flow_table
);
765 log_table_sz
= ilog2(ft
->max_fte
);
766 next_ft
= find_next_chained_ft(fs_prio
);
767 err
= mlx5_cmd_create_flow_table(root
->dev
, ft
->vport
, ft
->type
, ft
->level
,
768 log_table_sz
, next_ft
, &ft
->id
);
772 err
= connect_flow_table(root
->dev
, ft
, fs_prio
);
775 lock_ref_node(&fs_prio
->node
);
776 tree_add_node(&ft
->node
, &fs_prio
->node
);
777 list_add_flow_table(ft
, fs_prio
);
779 unlock_ref_node(&fs_prio
->node
);
780 mutex_unlock(&root
->chain_lock
);
783 mlx5_cmd_destroy_flow_table(root
->dev
, ft
);
787 mutex_unlock(&root
->chain_lock
);
791 struct mlx5_flow_table
*mlx5_create_flow_table(struct mlx5_flow_namespace
*ns
,
792 int prio
, int max_fte
,
795 return __mlx5_create_flow_table(ns
, 0, prio
, max_fte
, level
);
798 struct mlx5_flow_table
*mlx5_create_vport_flow_table(struct mlx5_flow_namespace
*ns
,
799 int prio
, int max_fte
,
800 u32 level
, u16 vport
)
802 return __mlx5_create_flow_table(ns
, vport
, prio
, max_fte
, level
);
805 struct mlx5_flow_table
*mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace
*ns
,
807 int num_flow_table_entries
,
811 struct mlx5_flow_table
*ft
;
813 if (max_num_groups
> num_flow_table_entries
)
814 return ERR_PTR(-EINVAL
);
816 ft
= mlx5_create_flow_table(ns
, prio
, num_flow_table_entries
, level
);
820 ft
->autogroup
.active
= true;
821 ft
->autogroup
.required_groups
= max_num_groups
;
825 EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table
);
827 /* Flow table should be locked */
828 static struct mlx5_flow_group
*create_flow_group_common(struct mlx5_flow_table
*ft
,
834 struct mlx5_flow_group
*fg
;
835 struct mlx5_core_dev
*dev
= get_dev(&ft
->node
);
839 return ERR_PTR(-ENODEV
);
841 fg
= alloc_flow_group(fg_in
);
845 err
= mlx5_cmd_create_flow_group(dev
, ft
, fg_in
, &fg
->id
);
851 if (ft
->autogroup
.active
)
852 ft
->autogroup
.num_groups
++;
853 /* Add node to tree */
854 tree_init_node(&fg
->node
, !is_auto_fg
, del_flow_group
);
855 tree_add_node(&fg
->node
, &ft
->node
);
856 /* Add node to group list */
857 list_add(&fg
->node
.list
, ft
->node
.children
.prev
);
862 struct mlx5_flow_group
*mlx5_create_flow_group(struct mlx5_flow_table
*ft
,
865 struct mlx5_flow_group
*fg
;
867 if (ft
->autogroup
.active
)
868 return ERR_PTR(-EPERM
);
870 lock_ref_node(&ft
->node
);
871 fg
= create_flow_group_common(ft
, fg_in
, &ft
->node
.children
, false);
872 unlock_ref_node(&ft
->node
);
877 static struct mlx5_flow_rule
*alloc_rule(struct mlx5_flow_destination
*dest
)
879 struct mlx5_flow_rule
*rule
;
881 rule
= kzalloc(sizeof(*rule
), GFP_KERNEL
);
885 INIT_LIST_HEAD(&rule
->next_ft
);
886 rule
->node
.type
= FS_TYPE_FLOW_DEST
;
888 memcpy(&rule
->dest_attr
, dest
, sizeof(*dest
));
893 /* fte should not be deleted while calling this function */
894 static struct mlx5_flow_rule
*add_rule_fte(struct fs_fte
*fte
,
895 struct mlx5_flow_group
*fg
,
896 struct mlx5_flow_destination
*dest
)
898 struct mlx5_flow_table
*ft
;
899 struct mlx5_flow_rule
*rule
;
903 rule
= alloc_rule(dest
);
905 return ERR_PTR(-ENOMEM
);
907 fs_get_obj(ft
, fg
->node
.parent
);
908 /* Add dest to dests list- we need flow tables to be in the
909 * end of the list for forward to next prio rules.
911 tree_init_node(&rule
->node
, 1, del_rule
);
912 if (dest
&& dest
->type
!= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
)
913 list_add(&rule
->node
.list
, &fte
->node
.children
);
915 list_add_tail(&rule
->node
.list
, &fte
->node
.children
);
919 modify_mask
|= dest
->type
== MLX5_FLOW_DESTINATION_TYPE_COUNTER
?
920 BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS
) :
921 BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
);
924 if (fte
->dests_size
== 1 || !dest
)
925 err
= mlx5_cmd_create_fte(get_dev(&ft
->node
),
928 err
= mlx5_cmd_update_fte(get_dev(&ft
->node
),
929 ft
, fg
->id
, modify_mask
, fte
);
933 fte
->status
|= FS_FTE_STATUS_EXISTING
;
938 list_del(&rule
->node
.list
);
945 /* Assumed fg is locked */
946 static unsigned int get_free_fte_index(struct mlx5_flow_group
*fg
,
947 struct list_head
**prev
)
950 unsigned int start
= fg
->start_index
;
953 *prev
= &fg
->node
.children
;
955 /* assumed list is sorted by index */
956 fs_for_each_fte(fte
, fg
) {
957 if (fte
->index
!= start
)
961 *prev
= &fte
->node
.list
;
967 /* prev is output, prev->next = new_fte */
968 static struct fs_fte
*create_fte(struct mlx5_flow_group
*fg
,
972 struct list_head
**prev
)
977 index
= get_free_fte_index(fg
, prev
);
978 fte
= alloc_fte(action
, flow_tag
, match_value
, index
);
985 static struct mlx5_flow_group
*create_autogroup(struct mlx5_flow_table
*ft
,
986 u8 match_criteria_enable
,
989 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
990 struct list_head
*prev
= &ft
->node
.children
;
991 unsigned int candidate_index
= 0;
992 struct mlx5_flow_group
*fg
;
993 void *match_criteria_addr
;
994 unsigned int group_size
= 0;
997 if (!ft
->autogroup
.active
)
998 return ERR_PTR(-ENOENT
);
1000 in
= mlx5_vzalloc(inlen
);
1002 return ERR_PTR(-ENOMEM
);
1004 if (ft
->autogroup
.num_groups
< ft
->autogroup
.required_groups
)
1005 /* We save place for flow groups in addition to max types */
1006 group_size
= ft
->max_fte
/ (ft
->autogroup
.required_groups
+ 1);
1008 /* ft->max_fte == ft->autogroup.max_types */
1009 if (group_size
== 0)
1012 /* sorted by start_index */
1013 fs_for_each_fg(fg
, ft
) {
1014 if (candidate_index
+ group_size
> fg
->start_index
)
1015 candidate_index
= fg
->start_index
+ fg
->max_ftes
;
1018 prev
= &fg
->node
.list
;
1021 if (candidate_index
+ group_size
> ft
->max_fte
) {
1022 fg
= ERR_PTR(-ENOSPC
);
1026 MLX5_SET(create_flow_group_in
, in
, match_criteria_enable
,
1027 match_criteria_enable
);
1028 MLX5_SET(create_flow_group_in
, in
, start_flow_index
, candidate_index
);
1029 MLX5_SET(create_flow_group_in
, in
, end_flow_index
, candidate_index
+
1031 match_criteria_addr
= MLX5_ADDR_OF(create_flow_group_in
,
1032 in
, match_criteria
);
1033 memcpy(match_criteria_addr
, match_criteria
,
1034 MLX5_ST_SZ_BYTES(fte_match_param
));
1036 fg
= create_flow_group_common(ft
, in
, prev
, true);
1042 static struct mlx5_flow_rule
*find_flow_rule(struct fs_fte
*fte
,
1043 struct mlx5_flow_destination
*dest
)
1045 struct mlx5_flow_rule
*rule
;
1047 list_for_each_entry(rule
, &fte
->node
.children
, node
.list
) {
1048 if (rule
->dest_attr
.type
== dest
->type
) {
1049 if ((dest
->type
== MLX5_FLOW_DESTINATION_TYPE_VPORT
&&
1050 dest
->vport_num
== rule
->dest_attr
.vport_num
) ||
1051 (dest
->type
== MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
&&
1052 dest
->ft
== rule
->dest_attr
.ft
) ||
1053 (dest
->type
== MLX5_FLOW_DESTINATION_TYPE_TIR
&&
1054 dest
->tir_num
== rule
->dest_attr
.tir_num
))
1061 static struct mlx5_flow_rule
*add_rule_fg(struct mlx5_flow_group
*fg
,
1065 struct mlx5_flow_destination
*dest
)
1068 struct mlx5_flow_rule
*rule
;
1069 struct mlx5_flow_table
*ft
;
1070 struct list_head
*prev
;
1072 nested_lock_ref_node(&fg
->node
, FS_MUTEX_PARENT
);
1073 fs_for_each_fte(fte
, fg
) {
1074 nested_lock_ref_node(&fte
->node
, FS_MUTEX_CHILD
);
1075 if (compare_match_value(&fg
->mask
, match_value
, &fte
->val
) &&
1076 action
== fte
->action
&& flow_tag
== fte
->flow_tag
) {
1077 rule
= find_flow_rule(fte
, dest
);
1079 atomic_inc(&rule
->node
.refcount
);
1080 unlock_ref_node(&fte
->node
);
1081 unlock_ref_node(&fg
->node
);
1084 rule
= add_rule_fte(fte
, fg
, dest
);
1085 unlock_ref_node(&fte
->node
);
1091 unlock_ref_node(&fte
->node
);
1093 fs_get_obj(ft
, fg
->node
.parent
);
1094 if (fg
->num_ftes
>= fg
->max_ftes
) {
1095 rule
= ERR_PTR(-ENOSPC
);
1099 fte
= create_fte(fg
, match_value
, action
, flow_tag
, &prev
);
1104 tree_init_node(&fte
->node
, 0, del_fte
);
1105 rule
= add_rule_fte(fte
, fg
, dest
);
1113 tree_add_node(&fte
->node
, &fg
->node
);
1114 list_add(&fte
->node
.list
, prev
);
1116 tree_add_node(&rule
->node
, &fte
->node
);
1118 unlock_ref_node(&fg
->node
);
1122 struct mlx5_fc
*mlx5_flow_rule_counter(struct mlx5_flow_rule
*rule
)
1124 struct mlx5_flow_rule
*dst
;
1127 fs_get_obj(fte
, rule
->node
.parent
);
1129 fs_for_each_dst(dst
, fte
) {
1130 if (dst
->dest_attr
.type
== MLX5_FLOW_DESTINATION_TYPE_COUNTER
)
1131 return dst
->dest_attr
.counter
;
1137 static bool counter_is_valid(struct mlx5_fc
*counter
, u32 action
)
1139 if (!(action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
))
1145 /* Hardware support counter for a drop action only */
1146 return action
== (MLX5_FLOW_CONTEXT_ACTION_DROP
| MLX5_FLOW_CONTEXT_ACTION_COUNT
);
1149 static bool dest_is_valid(struct mlx5_flow_destination
*dest
,
1151 struct mlx5_flow_table
*ft
)
1153 if (dest
&& (dest
->type
== MLX5_FLOW_DESTINATION_TYPE_COUNTER
))
1154 return counter_is_valid(dest
->counter
, action
);
1156 if (!(action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
))
1159 if (!dest
|| ((dest
->type
==
1160 MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
) &&
1161 (dest
->ft
->level
<= ft
->level
)))
1166 static struct mlx5_flow_rule
*
1167 _mlx5_add_flow_rule(struct mlx5_flow_table
*ft
,
1168 struct mlx5_flow_spec
*spec
,
1171 struct mlx5_flow_destination
*dest
)
1173 struct mlx5_flow_group
*g
;
1174 struct mlx5_flow_rule
*rule
;
1176 if (!dest_is_valid(dest
, action
, ft
))
1177 return ERR_PTR(-EINVAL
);
1179 nested_lock_ref_node(&ft
->node
, FS_MUTEX_GRANDPARENT
);
1180 fs_for_each_fg(g
, ft
)
1181 if (compare_match_criteria(g
->mask
.match_criteria_enable
,
1182 spec
->match_criteria_enable
,
1183 g
->mask
.match_criteria
,
1184 spec
->match_criteria
)) {
1185 rule
= add_rule_fg(g
, spec
->match_value
,
1186 action
, flow_tag
, dest
);
1187 if (!IS_ERR(rule
) || PTR_ERR(rule
) != -ENOSPC
)
1191 g
= create_autogroup(ft
, spec
->match_criteria_enable
,
1192 spec
->match_criteria
);
1198 rule
= add_rule_fg(g
, spec
->match_value
,
1199 action
, flow_tag
, dest
);
1201 /* Remove assumes refcount > 0 and autogroup creates a group
1202 * with a refcount = 0.
1204 unlock_ref_node(&ft
->node
);
1205 tree_get_node(&g
->node
);
1206 tree_remove_node(&g
->node
);
1210 unlock_ref_node(&ft
->node
);
1214 static bool fwd_next_prio_supported(struct mlx5_flow_table
*ft
)
1216 return ((ft
->type
== FS_FT_NIC_RX
) &&
1217 (MLX5_CAP_FLOWTABLE(get_dev(&ft
->node
), nic_rx_multi_path_tirs
)));
1220 struct mlx5_flow_rule
*
1221 mlx5_add_flow_rule(struct mlx5_flow_table
*ft
,
1222 struct mlx5_flow_spec
*spec
,
1225 struct mlx5_flow_destination
*dest
)
1227 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1228 struct mlx5_flow_destination gen_dest
;
1229 struct mlx5_flow_table
*next_ft
= NULL
;
1230 struct mlx5_flow_rule
*rule
= NULL
;
1231 u32 sw_action
= action
;
1232 struct fs_prio
*prio
;
1234 fs_get_obj(prio
, ft
->node
.parent
);
1235 if (action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
1236 if (!fwd_next_prio_supported(ft
))
1237 return ERR_PTR(-EOPNOTSUPP
);
1239 return ERR_PTR(-EINVAL
);
1240 mutex_lock(&root
->chain_lock
);
1241 next_ft
= find_next_chained_ft(prio
);
1243 gen_dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
1244 gen_dest
.ft
= next_ft
;
1246 action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1248 mutex_unlock(&root
->chain_lock
);
1249 return ERR_PTR(-EOPNOTSUPP
);
1253 rule
= _mlx5_add_flow_rule(ft
, spec
, action
, flow_tag
, dest
);
1255 if (sw_action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
1256 if (!IS_ERR_OR_NULL(rule
) &&
1257 (list_empty(&rule
->next_ft
))) {
1258 mutex_lock(&next_ft
->lock
);
1259 list_add(&rule
->next_ft
, &next_ft
->fwd_rules
);
1260 mutex_unlock(&next_ft
->lock
);
1261 rule
->sw_action
= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
;
1263 mutex_unlock(&root
->chain_lock
);
1267 EXPORT_SYMBOL(mlx5_add_flow_rule
);
1269 void mlx5_del_flow_rule(struct mlx5_flow_rule
*rule
)
1271 tree_remove_node(&rule
->node
);
1273 EXPORT_SYMBOL(mlx5_del_flow_rule
);
1275 /* Assuming prio->node.children(flow tables) is sorted by level */
1276 static struct mlx5_flow_table
*find_next_ft(struct mlx5_flow_table
*ft
)
1278 struct fs_prio
*prio
;
1280 fs_get_obj(prio
, ft
->node
.parent
);
1282 if (!list_is_last(&ft
->node
.list
, &prio
->node
.children
))
1283 return list_next_entry(ft
, node
.list
);
1284 return find_next_chained_ft(prio
);
1287 static int update_root_ft_destroy(struct mlx5_flow_table
*ft
)
1289 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1290 struct mlx5_flow_table
*new_root_ft
= NULL
;
1292 if (root
->root_ft
!= ft
)
1295 new_root_ft
= find_next_ft(ft
);
1297 int err
= mlx5_cmd_update_root_ft(root
->dev
, new_root_ft
);
1300 mlx5_core_warn(root
->dev
, "Update root flow table of id=%u failed\n",
1305 root
->root_ft
= new_root_ft
;
1309 /* Connect flow table from previous priority to
1310 * the next flow table.
1312 static int disconnect_flow_table(struct mlx5_flow_table
*ft
)
1314 struct mlx5_core_dev
*dev
= get_dev(&ft
->node
);
1315 struct mlx5_flow_table
*next_ft
;
1316 struct fs_prio
*prio
;
1319 err
= update_root_ft_destroy(ft
);
1323 fs_get_obj(prio
, ft
->node
.parent
);
1324 if (!(list_first_entry(&prio
->node
.children
,
1325 struct mlx5_flow_table
,
1329 next_ft
= find_next_chained_ft(prio
);
1330 err
= connect_fwd_rules(dev
, next_ft
, ft
);
1334 err
= connect_prev_fts(dev
, next_ft
, prio
);
1336 mlx5_core_warn(dev
, "Failed to disconnect flow table %d\n",
1341 int mlx5_destroy_flow_table(struct mlx5_flow_table
*ft
)
1343 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1346 mutex_lock(&root
->chain_lock
);
1347 err
= disconnect_flow_table(ft
);
1349 mutex_unlock(&root
->chain_lock
);
1352 if (tree_remove_node(&ft
->node
))
1353 mlx5_core_warn(get_dev(&ft
->node
), "Flow table %d wasn't destroyed, refcount > 1\n",
1355 mutex_unlock(&root
->chain_lock
);
1359 EXPORT_SYMBOL(mlx5_destroy_flow_table
);
1361 void mlx5_destroy_flow_group(struct mlx5_flow_group
*fg
)
1363 if (tree_remove_node(&fg
->node
))
1364 mlx5_core_warn(get_dev(&fg
->node
), "Flow group %d wasn't destroyed, refcount > 1\n",
1368 struct mlx5_flow_namespace
*mlx5_get_flow_namespace(struct mlx5_core_dev
*dev
,
1369 enum mlx5_flow_namespace_type type
)
1371 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
1372 struct mlx5_flow_root_namespace
*root_ns
;
1374 struct fs_prio
*fs_prio
;
1375 struct mlx5_flow_namespace
*ns
;
1381 case MLX5_FLOW_NAMESPACE_BYPASS
:
1382 case MLX5_FLOW_NAMESPACE_OFFLOADS
:
1383 case MLX5_FLOW_NAMESPACE_ETHTOOL
:
1384 case MLX5_FLOW_NAMESPACE_KERNEL
:
1385 case MLX5_FLOW_NAMESPACE_LEFTOVERS
:
1386 case MLX5_FLOW_NAMESPACE_ANCHOR
:
1389 case MLX5_FLOW_NAMESPACE_FDB
:
1390 if (steering
->fdb_root_ns
)
1391 return &steering
->fdb_root_ns
->ns
;
1394 case MLX5_FLOW_NAMESPACE_ESW_EGRESS
:
1395 if (steering
->esw_egress_root_ns
)
1396 return &steering
->esw_egress_root_ns
->ns
;
1399 case MLX5_FLOW_NAMESPACE_ESW_INGRESS
:
1400 if (steering
->esw_ingress_root_ns
)
1401 return &steering
->esw_ingress_root_ns
->ns
;
1408 root_ns
= steering
->root_ns
;
1412 fs_prio
= find_prio(&root_ns
->ns
, prio
);
1416 ns
= list_first_entry(&fs_prio
->node
.children
,
1422 EXPORT_SYMBOL(mlx5_get_flow_namespace
);
1424 static struct fs_prio
*fs_create_prio(struct mlx5_flow_namespace
*ns
,
1425 unsigned int prio
, int num_levels
)
1427 struct fs_prio
*fs_prio
;
1429 fs_prio
= kzalloc(sizeof(*fs_prio
), GFP_KERNEL
);
1431 return ERR_PTR(-ENOMEM
);
1433 fs_prio
->node
.type
= FS_TYPE_PRIO
;
1434 tree_init_node(&fs_prio
->node
, 1, NULL
);
1435 tree_add_node(&fs_prio
->node
, &ns
->node
);
1436 fs_prio
->num_levels
= num_levels
;
1437 fs_prio
->prio
= prio
;
1438 list_add_tail(&fs_prio
->node
.list
, &ns
->node
.children
);
1443 static struct mlx5_flow_namespace
*fs_init_namespace(struct mlx5_flow_namespace
1446 ns
->node
.type
= FS_TYPE_NAMESPACE
;
1451 static struct mlx5_flow_namespace
*fs_create_namespace(struct fs_prio
*prio
)
1453 struct mlx5_flow_namespace
*ns
;
1455 ns
= kzalloc(sizeof(*ns
), GFP_KERNEL
);
1457 return ERR_PTR(-ENOMEM
);
1459 fs_init_namespace(ns
);
1460 tree_init_node(&ns
->node
, 1, NULL
);
1461 tree_add_node(&ns
->node
, &prio
->node
);
1462 list_add_tail(&ns
->node
.list
, &prio
->node
.children
);
1467 static int create_leaf_prios(struct mlx5_flow_namespace
*ns
, int prio
,
1468 struct init_tree_node
*prio_metadata
)
1470 struct fs_prio
*fs_prio
;
1473 for (i
= 0; i
< prio_metadata
->num_leaf_prios
; i
++) {
1474 fs_prio
= fs_create_prio(ns
, prio
++, prio_metadata
->num_levels
);
1475 if (IS_ERR(fs_prio
))
1476 return PTR_ERR(fs_prio
);
1481 #define FLOW_TABLE_BIT_SZ 1
1482 #define GET_FLOW_TABLE_CAP(dev, offset) \
1483 ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
1485 (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
1486 static bool has_required_caps(struct mlx5_core_dev
*dev
, struct node_caps
*caps
)
1490 for (i
= 0; i
< caps
->arr_sz
; i
++) {
1491 if (!GET_FLOW_TABLE_CAP(dev
, caps
->caps
[i
]))
1497 static int init_root_tree_recursive(struct mlx5_flow_steering
*steering
,
1498 struct init_tree_node
*init_node
,
1499 struct fs_node
*fs_parent_node
,
1500 struct init_tree_node
*init_parent_node
,
1503 int max_ft_level
= MLX5_CAP_FLOWTABLE(steering
->dev
,
1504 flow_table_properties_nic_receive
.
1506 struct mlx5_flow_namespace
*fs_ns
;
1507 struct fs_prio
*fs_prio
;
1508 struct fs_node
*base
;
1512 if (init_node
->type
== FS_TYPE_PRIO
) {
1513 if ((init_node
->min_ft_level
> max_ft_level
) ||
1514 !has_required_caps(steering
->dev
, &init_node
->caps
))
1517 fs_get_obj(fs_ns
, fs_parent_node
);
1518 if (init_node
->num_leaf_prios
)
1519 return create_leaf_prios(fs_ns
, prio
, init_node
);
1520 fs_prio
= fs_create_prio(fs_ns
, prio
, init_node
->num_levels
);
1521 if (IS_ERR(fs_prio
))
1522 return PTR_ERR(fs_prio
);
1523 base
= &fs_prio
->node
;
1524 } else if (init_node
->type
== FS_TYPE_NAMESPACE
) {
1525 fs_get_obj(fs_prio
, fs_parent_node
);
1526 fs_ns
= fs_create_namespace(fs_prio
);
1528 return PTR_ERR(fs_ns
);
1529 base
= &fs_ns
->node
;
1534 for (i
= 0; i
< init_node
->ar_size
; i
++) {
1535 err
= init_root_tree_recursive(steering
, &init_node
->children
[i
],
1536 base
, init_node
, prio
);
1539 if (init_node
->children
[i
].type
== FS_TYPE_PRIO
&&
1540 init_node
->children
[i
].num_leaf_prios
) {
1541 prio
+= init_node
->children
[i
].num_leaf_prios
;
1548 static int init_root_tree(struct mlx5_flow_steering
*steering
,
1549 struct init_tree_node
*init_node
,
1550 struct fs_node
*fs_parent_node
)
1553 struct mlx5_flow_namespace
*fs_ns
;
1556 fs_get_obj(fs_ns
, fs_parent_node
);
1557 for (i
= 0; i
< init_node
->ar_size
; i
++) {
1558 err
= init_root_tree_recursive(steering
, &init_node
->children
[i
],
1567 static struct mlx5_flow_root_namespace
*create_root_ns(struct mlx5_flow_steering
*steering
,
1568 enum fs_flow_table_type
1571 struct mlx5_flow_root_namespace
*root_ns
;
1572 struct mlx5_flow_namespace
*ns
;
1574 /* Create the root namespace */
1575 root_ns
= mlx5_vzalloc(sizeof(*root_ns
));
1579 root_ns
->dev
= steering
->dev
;
1580 root_ns
->table_type
= table_type
;
1583 fs_init_namespace(ns
);
1584 mutex_init(&root_ns
->chain_lock
);
1585 tree_init_node(&ns
->node
, 1, NULL
);
1586 tree_add_node(&ns
->node
, NULL
);
1591 static void set_prio_attrs_in_prio(struct fs_prio
*prio
, int acc_level
);
1593 static int set_prio_attrs_in_ns(struct mlx5_flow_namespace
*ns
, int acc_level
)
1595 struct fs_prio
*prio
;
1597 fs_for_each_prio(prio
, ns
) {
1598 /* This updates prio start_level and num_levels */
1599 set_prio_attrs_in_prio(prio
, acc_level
);
1600 acc_level
+= prio
->num_levels
;
1605 static void set_prio_attrs_in_prio(struct fs_prio
*prio
, int acc_level
)
1607 struct mlx5_flow_namespace
*ns
;
1608 int acc_level_ns
= acc_level
;
1610 prio
->start_level
= acc_level
;
1611 fs_for_each_ns(ns
, prio
)
1612 /* This updates start_level and num_levels of ns's priority descendants */
1613 acc_level_ns
= set_prio_attrs_in_ns(ns
, acc_level
);
1614 if (!prio
->num_levels
)
1615 prio
->num_levels
= acc_level_ns
- prio
->start_level
;
1616 WARN_ON(prio
->num_levels
< acc_level_ns
- prio
->start_level
);
1619 static void set_prio_attrs(struct mlx5_flow_root_namespace
*root_ns
)
1621 struct mlx5_flow_namespace
*ns
= &root_ns
->ns
;
1622 struct fs_prio
*prio
;
1623 int start_level
= 0;
1625 fs_for_each_prio(prio
, ns
) {
1626 set_prio_attrs_in_prio(prio
, start_level
);
1627 start_level
+= prio
->num_levels
;
1631 #define ANCHOR_PRIO 0
1632 #define ANCHOR_SIZE 1
1633 #define ANCHOR_LEVEL 0
1634 static int create_anchor_flow_table(struct mlx5_flow_steering
*steering
)
1636 struct mlx5_flow_namespace
*ns
= NULL
;
1637 struct mlx5_flow_table
*ft
;
1639 ns
= mlx5_get_flow_namespace(steering
->dev
, MLX5_FLOW_NAMESPACE_ANCHOR
);
1642 ft
= mlx5_create_flow_table(ns
, ANCHOR_PRIO
, ANCHOR_SIZE
, ANCHOR_LEVEL
);
1644 mlx5_core_err(steering
->dev
, "Failed to create last anchor flow table");
1650 static int init_root_ns(struct mlx5_flow_steering
*steering
)
1653 steering
->root_ns
= create_root_ns(steering
, FS_FT_NIC_RX
);
1654 if (IS_ERR_OR_NULL(steering
->root_ns
))
1657 if (init_root_tree(steering
, &root_fs
, &steering
->root_ns
->ns
.node
))
1660 set_prio_attrs(steering
->root_ns
);
1662 if (create_anchor_flow_table(steering
))
1668 mlx5_cleanup_fs(steering
->dev
);
1672 static void clean_tree(struct fs_node
*node
)
1675 struct fs_node
*iter
;
1676 struct fs_node
*temp
;
1678 list_for_each_entry_safe(iter
, temp
, &node
->children
, list
)
1680 tree_remove_node(node
);
1684 static void cleanup_root_ns(struct mlx5_flow_root_namespace
*root_ns
)
1689 clean_tree(&root_ns
->ns
.node
);
1692 void mlx5_cleanup_fs(struct mlx5_core_dev
*dev
)
1694 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
1696 if (MLX5_CAP_GEN(dev
, port_type
) != MLX5_CAP_PORT_TYPE_ETH
)
1699 cleanup_root_ns(steering
->root_ns
);
1700 cleanup_root_ns(steering
->esw_egress_root_ns
);
1701 cleanup_root_ns(steering
->esw_ingress_root_ns
);
1702 cleanup_root_ns(steering
->fdb_root_ns
);
1703 mlx5_cleanup_fc_stats(dev
);
1707 static int init_fdb_root_ns(struct mlx5_flow_steering
*steering
)
1709 struct fs_prio
*prio
;
1711 steering
->fdb_root_ns
= create_root_ns(steering
, FS_FT_FDB
);
1712 if (!steering
->fdb_root_ns
)
1715 prio
= fs_create_prio(&steering
->fdb_root_ns
->ns
, 0, 1);
1719 prio
= fs_create_prio(&steering
->fdb_root_ns
->ns
, 1, 1);
1723 set_prio_attrs(steering
->fdb_root_ns
);
1727 cleanup_root_ns(steering
->fdb_root_ns
);
1728 steering
->fdb_root_ns
= NULL
;
1729 return PTR_ERR(prio
);
1732 static int init_ingress_acl_root_ns(struct mlx5_flow_steering
*steering
)
1734 struct fs_prio
*prio
;
1736 steering
->esw_egress_root_ns
= create_root_ns(steering
, FS_FT_ESW_EGRESS_ACL
);
1737 if (!steering
->esw_egress_root_ns
)
1741 prio
= fs_create_prio(&steering
->esw_egress_root_ns
->ns
, 0,
1742 MLX5_TOTAL_VPORTS(steering
->dev
));
1743 return PTR_ERR_OR_ZERO(prio
);
1746 static int init_egress_acl_root_ns(struct mlx5_flow_steering
*steering
)
1748 struct fs_prio
*prio
;
1750 steering
->esw_ingress_root_ns
= create_root_ns(steering
, FS_FT_ESW_INGRESS_ACL
);
1751 if (!steering
->esw_ingress_root_ns
)
1755 prio
= fs_create_prio(&steering
->esw_ingress_root_ns
->ns
, 0,
1756 MLX5_TOTAL_VPORTS(steering
->dev
));
1757 return PTR_ERR_OR_ZERO(prio
);
1760 int mlx5_init_fs(struct mlx5_core_dev
*dev
)
1762 struct mlx5_flow_steering
*steering
;
1765 if (MLX5_CAP_GEN(dev
, port_type
) != MLX5_CAP_PORT_TYPE_ETH
)
1768 err
= mlx5_init_fc_stats(dev
);
1772 steering
= kzalloc(sizeof(*steering
), GFP_KERNEL
);
1775 steering
->dev
= dev
;
1776 dev
->priv
.steering
= steering
;
1778 if (MLX5_CAP_GEN(dev
, nic_flow_table
) &&
1779 MLX5_CAP_FLOWTABLE_NIC_RX(dev
, ft_support
)) {
1780 err
= init_root_ns(steering
);
1785 if (MLX5_CAP_GEN(dev
, eswitch_flow_table
)) {
1786 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, ft_support
)) {
1787 err
= init_fdb_root_ns(steering
);
1791 if (MLX5_CAP_ESW_EGRESS_ACL(dev
, ft_support
)) {
1792 err
= init_egress_acl_root_ns(steering
);
1796 if (MLX5_CAP_ESW_INGRESS_ACL(dev
, ft_support
)) {
1797 err
= init_ingress_acl_root_ns(steering
);
1805 mlx5_cleanup_fs(dev
);