1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/mlx5.h"
47 #include "lib/devcom.h"
48 #include "lib/eq.h"
49 #include "lib/fs_chains.h"
50 #include "en_tc.h"
51 #include "en/mapping.h"
52 #include "devlink.h"
53 #include "lag/lag.h"
54 #include "en/tc/post_meter.h"
55 #include "fw_reset.h"
56
57 /* There are two match-all miss flows, one for unicast dst mac and
58 * one for multicast.
59 */
60 #define MLX5_ESW_MISS_FLOWS (2)
61 #define UPLINK_REP_INDEX 0
62
63 #define MLX5_ESW_VPORT_TBL_SIZE 128
64 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
65
66 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
67
68 #define MLX5_ESW_MAX_CTRL_EQS 4
69 #define MLX5_ESW_DEFAULT_SF_COMP_EQS 8
70
71 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
72 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
73 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
74 .flags = 0,
75 };
76
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)77 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
78 u16 vport_num)
79 {
80 return xa_load(&esw->offloads.vport_reps, vport_num);
81 }
82
83 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)84 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
85 struct mlx5_flow_spec *spec,
86 struct mlx5_esw_flow_attr *attr)
87 {
88 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
89 return;
90
91 if (attr->int_port) {
92 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
93
94 return;
95 }
96
97 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
98 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
99 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
100 }
101
102 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
103 * are not needed as well in the following process. So clear them all for simplicity.
104 */
105 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)106 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
107 {
108 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
109 void *misc2;
110
111 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
112 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
113
114 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
115 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
116
117 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
118 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
119 }
120 }
121
122 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)123 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
124 struct mlx5_flow_spec *spec,
125 struct mlx5_flow_attr *attr,
126 struct mlx5_eswitch *src_esw,
127 u16 vport)
128 {
129 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
130 u32 metadata;
131 void *misc2;
132 void *misc;
133
134 /* Use metadata matching because vport is not represented by single
135 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
136 */
137 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
138 if (mlx5_esw_indir_table_decap_vport(attr))
139 vport = mlx5_esw_indir_table_decap_vport(attr);
140
141 if (!attr->chain && esw_attr && esw_attr->int_port)
142 metadata =
143 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
144 else
145 metadata =
146 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
147
148 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
149 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
150
151 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
152 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
153 mlx5_eswitch_get_vport_metadata_mask());
154
155 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
156 } else {
157 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
158 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
159
160 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
161 MLX5_SET(fte_match_set_misc, misc,
162 source_eswitch_owner_vhca_id,
163 MLX5_CAP_GEN(src_esw->dev, vhca_id));
164
165 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
166 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
167 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
168 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
169 source_eswitch_owner_vhca_id);
170
171 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
172 }
173 }
174
175 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)176 esw_setup_decap_indir(struct mlx5_eswitch *esw,
177 struct mlx5_flow_attr *attr)
178 {
179 struct mlx5_flow_table *ft;
180
181 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
182 return -EOPNOTSUPP;
183
184 ft = mlx5_esw_indir_table_get(esw, attr,
185 mlx5_esw_indir_table_decap_vport(attr), true);
186 return PTR_ERR_OR_ZERO(ft);
187 }
188
189 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)190 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
191 struct mlx5_flow_attr *attr)
192 {
193 if (mlx5_esw_indir_table_decap_vport(attr))
194 mlx5_esw_indir_table_put(esw,
195 mlx5_esw_indir_table_decap_vport(attr),
196 true);
197 }
198
199 static int
esw_setup_mtu_dest(struct mlx5_flow_destination * dest,struct mlx5e_meter_attr * meter,int i)200 esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
201 struct mlx5e_meter_attr *meter,
202 int i)
203 {
204 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
205 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
206 dest[i].range.min = 0;
207 dest[i].range.max = meter->params.mtu;
208 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter);
209 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter);
210
211 return 0;
212 }
213
214 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)215 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
216 struct mlx5_flow_act *flow_act,
217 u32 sampler_id,
218 int i)
219 {
220 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
221 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
222 dest[i].sampler_id = sampler_id;
223
224 return 0;
225 }
226
227 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int i)228 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
229 struct mlx5_flow_act *flow_act,
230 struct mlx5_eswitch *esw,
231 struct mlx5_flow_attr *attr,
232 int i)
233 {
234 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
235 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
236 dest[i].ft = attr->dest_ft;
237
238 if (mlx5_esw_indir_table_decap_vport(attr))
239 return esw_setup_decap_indir(esw, attr);
240 return 0;
241 }
242
243 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)244 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
245 struct mlx5_fs_chains *chains, int i)
246 {
247 if (mlx5_chains_ignore_flow_level_supported(chains))
248 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
249 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
250 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
251 }
252
253 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)254 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
255 struct mlx5_eswitch *esw, int i)
256 {
257 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
258 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
259 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
260 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
261 }
262
263 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)264 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
265 struct mlx5_flow_act *flow_act,
266 struct mlx5_fs_chains *chains,
267 u32 chain, u32 prio, u32 level,
268 int i)
269 {
270 struct mlx5_flow_table *ft;
271
272 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
273 ft = mlx5_chains_get_table(chains, chain, prio, level);
274 if (IS_ERR(ft))
275 return PTR_ERR(ft);
276
277 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
278 dest[i].ft = ft;
279 return 0;
280 }
281
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)282 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
283 int from, int to)
284 {
285 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
286 struct mlx5_fs_chains *chains = esw_chains(esw);
287 int i;
288
289 for (i = from; i < to; i++)
290 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
291 mlx5_chains_put_table(chains, 0, 1, 0);
292 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
293 esw_attr->dests[i].mdev))
294 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false);
295 }
296
297 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)298 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
299 {
300 int i;
301
302 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
303 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
304 return true;
305 return false;
306 }
307
308 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)309 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
310 struct mlx5_flow_act *flow_act,
311 struct mlx5_eswitch *esw,
312 struct mlx5_fs_chains *chains,
313 struct mlx5_flow_attr *attr,
314 int *i)
315 {
316 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
317 int err;
318
319 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
320 return -EOPNOTSUPP;
321
322 /* flow steering cannot handle more than one dest with the same ft
323 * in a single flow
324 */
325 if (esw_attr->out_count - esw_attr->split_count > 1)
326 return -EOPNOTSUPP;
327
328 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
329 if (err)
330 return err;
331
332 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
333 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
334 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
335 }
336 (*i)++;
337
338 return 0;
339 }
340
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)341 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
342 struct mlx5_flow_attr *attr)
343 {
344 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
345
346 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
347 }
348
349 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)350 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
351 {
352 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
353 bool result = false;
354 int i;
355
356 /* Indirect table is supported only for flows with in_port uplink
357 * and the destination is vport on the same eswitch as the uplink,
358 * return false in case at least one of destinations doesn't meet
359 * this criteria.
360 */
361 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
362 if (esw_attr->dests[i].vport_valid &&
363 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
364 esw_attr->dests[i].mdev)) {
365 result = true;
366 } else {
367 result = false;
368 break;
369 }
370 }
371 return result;
372 }
373
374 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int * i)375 esw_setup_indir_table(struct mlx5_flow_destination *dest,
376 struct mlx5_flow_act *flow_act,
377 struct mlx5_eswitch *esw,
378 struct mlx5_flow_attr *attr,
379 int *i)
380 {
381 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
382 int j, err;
383
384 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
385 return -EOPNOTSUPP;
386
387 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
388 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
389 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
390
391 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
392 esw_attr->dests[j].vport, false);
393 if (IS_ERR(dest[*i].ft)) {
394 err = PTR_ERR(dest[*i].ft);
395 goto err_indir_tbl_get;
396 }
397 }
398
399 if (mlx5_esw_indir_table_decap_vport(attr)) {
400 err = esw_setup_decap_indir(esw, attr);
401 if (err)
402 goto err_indir_tbl_get;
403 }
404
405 return 0;
406
407 err_indir_tbl_get:
408 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
409 return err;
410 }
411
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)412 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
413 {
414 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
415
416 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
417 esw_cleanup_decap_indir(esw, attr);
418 }
419
420 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)421 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
422 {
423 mlx5_chains_put_table(chains, chain, prio, level);
424 }
425
esw_same_vhca_id(struct mlx5_core_dev * mdev1,struct mlx5_core_dev * mdev2)426 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
427 {
428 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
429 }
430
esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)431 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
432 struct mlx5_esw_flow_attr *esw_attr,
433 int attr_idx)
434 {
435 if (esw->offloads.ft_ipsec_tx_pol &&
436 esw_attr->dests[attr_idx].vport_valid &&
437 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
438 /* To be aligned with software, encryption is needed only for tunnel device */
439 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
440 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
441 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev))
442 return true;
443
444 return false;
445 }
446
esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)447 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
448 struct mlx5_esw_flow_attr *esw_attr)
449 {
450 int i;
451
452 if (!esw->offloads.ft_ipsec_tx_pol)
453 return true;
454
455 for (i = 0; i < esw_attr->split_count; i++)
456 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i))
457 return false;
458
459 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
460 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) &&
461 (esw_attr->out_count - esw_attr->split_count > 1))
462 return false;
463
464 return true;
465 }
466
467 static void
esw_setup_dest_fwd_vport(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)468 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
469 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
470 int attr_idx, int dest_idx, bool pkt_reformat)
471 {
472 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
473 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
474 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
475 dest[dest_idx].vport.vhca_id =
476 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
477 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
478 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
479 mlx5_lag_is_mpesw(esw->dev))
480 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
481 }
482 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
483 if (pkt_reformat) {
484 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
485 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
486 }
487 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
488 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
489 }
490 }
491
492 static void
esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)493 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
494 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
495 int attr_idx, int dest_idx, bool pkt_reformat)
496 {
497 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
498 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
499 if (pkt_reformat &&
500 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
501 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
502 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
503 }
504 }
505
506 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)507 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
508 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
509 int attr_idx, int dest_idx, bool pkt_reformat)
510 {
511 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
512 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
513 attr_idx, dest_idx, pkt_reformat);
514 else
515 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
516 attr_idx, dest_idx, pkt_reformat);
517 }
518
519 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)520 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
521 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
522 int i)
523 {
524 int j;
525
526 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
527 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
528 return i;
529 }
530
531 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)532 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
533 {
534 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
535 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
536 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
537 }
538
539 static bool
esw_dests_to_int_external(struct mlx5_flow_destination * dests,int max_dest)540 esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest)
541 {
542 bool internal_dest = false, external_dest = false;
543 int i;
544
545 for (i = 0; i < max_dest; i++) {
546 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT &&
547 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK)
548 continue;
549
550 /* Uplink dest is external, but considered as internal
551 * if there is reformat because firmware uses LB+hairpin to support it.
552 */
553 if (dests[i].vport.num == MLX5_VPORT_UPLINK &&
554 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID))
555 external_dest = true;
556 else
557 internal_dest = true;
558
559 if (internal_dest && external_dest)
560 return true;
561 }
562
563 return false;
564 }
565
566 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)567 esw_setup_dests(struct mlx5_flow_destination *dest,
568 struct mlx5_flow_act *flow_act,
569 struct mlx5_eswitch *esw,
570 struct mlx5_flow_attr *attr,
571 struct mlx5_flow_spec *spec,
572 int *i)
573 {
574 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
575 struct mlx5_fs_chains *chains = esw_chains(esw);
576 int err = 0;
577
578 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
579 esw_src_port_rewrite_supported(esw))
580 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
581
582 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
583 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
584 (*i)++;
585 goto out;
586 }
587
588 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
589 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
590 (*i)++;
591 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
592 esw_setup_accept_dest(dest, flow_act, chains, *i);
593 (*i)++;
594 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
595 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i);
596 (*i)++;
597 } else if (esw_is_indir_table(esw, attr)) {
598 err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
599 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
600 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
601 } else {
602 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
603
604 if (attr->dest_ft) {
605 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i);
606 (*i)++;
607 } else if (attr->dest_chain) {
608 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
609 1, 0, *i);
610 (*i)++;
611 }
612 }
613
614 if (attr->extra_split_ft) {
615 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
616 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
617 dest[*i].ft = attr->extra_split_ft;
618 (*i)++;
619 }
620
621 out:
622 return err;
623 }
624
625 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)626 esw_cleanup_dests(struct mlx5_eswitch *esw,
627 struct mlx5_flow_attr *attr)
628 {
629 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
630 struct mlx5_fs_chains *chains = esw_chains(esw);
631
632 if (attr->dest_ft) {
633 esw_cleanup_decap_indir(esw, attr);
634 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
635 if (attr->dest_chain)
636 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
637 else if (esw_is_indir_table(esw, attr))
638 esw_cleanup_indir_table(esw, attr);
639 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
640 esw_cleanup_chain_src_port_rewrite(esw, attr);
641 }
642 }
643
644 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)645 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
646 {
647 struct mlx5e_flow_meter_handle *meter;
648
649 meter = attr->meter_attr.meter;
650 flow_act->exe_aso.type = attr->exe_aso_type;
651 flow_act->exe_aso.object_id = meter->obj_id;
652 flow_act->exe_aso.base_id = mlx5e_flow_meter_get_base_id(meter);
653 flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
654 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
655 /* use metadata reg 5 for packet color */
656 flow_act->exe_aso.return_reg_id = 5;
657 }
658
659 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)660 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
661 struct mlx5_flow_spec *spec,
662 struct mlx5_flow_attr *attr)
663 {
664 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
665 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
666 struct mlx5_fs_chains *chains = esw_chains(esw);
667 bool split = !!(esw_attr->split_count);
668 struct mlx5_vport_tbl_attr fwd_attr;
669 struct mlx5_flow_destination *dest;
670 struct mlx5_flow_handle *rule;
671 struct mlx5_flow_table *fdb;
672 int i = 0;
673
674 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
675 return ERR_PTR(-EOPNOTSUPP);
676
677 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
678 return ERR_PTR(-EOPNOTSUPP);
679
680 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
681 return ERR_PTR(-EOPNOTSUPP);
682
683 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1);
684 if (!dest)
685 return ERR_PTR(-ENOMEM);
686
687 flow_act.action = attr->action;
688
689 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
690 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
691 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
692 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
693 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
694 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
695 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
696 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
697 }
698 }
699
700 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
701
702 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
703 int err;
704
705 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
706 if (err) {
707 rule = ERR_PTR(err);
708 goto err_create_goto_table;
709 }
710
711 /* Header rewrite with combined wire+loopback in FDB is not allowed */
712 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
713 esw_dests_to_int_external(dest, i)) {
714 esw_warn(esw->dev,
715 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n");
716 rule = ERR_PTR(-EINVAL);
717 goto err_esw_get;
718 }
719 }
720
721 if (esw_attr->decap_pkt_reformat)
722 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
723
724 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
725 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
726 dest[i].counter = attr->counter;
727 i++;
728 }
729
730 if (attr->outer_match_level != MLX5_MATCH_NONE)
731 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
732 if (attr->inner_match_level != MLX5_MATCH_NONE)
733 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
734
735 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
736 flow_act.modify_hdr = attr->modify_hdr;
737
738 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
739 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
740 esw_setup_meter(attr, &flow_act);
741
742 if (split) {
743 fwd_attr.chain = attr->chain;
744 fwd_attr.prio = attr->prio;
745 fwd_attr.vport = esw_attr->in_rep->vport;
746 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
747
748 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
749 } else {
750 if (attr->chain || attr->prio)
751 fdb = mlx5_chains_get_table(chains, attr->chain,
752 attr->prio, 0);
753 else
754 fdb = attr->ft;
755
756 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
757 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
758 esw_attr->in_mdev->priv.eswitch,
759 esw_attr->in_rep->vport);
760 }
761 if (IS_ERR(fdb)) {
762 rule = ERR_CAST(fdb);
763 goto err_esw_get;
764 }
765
766 if (!i) {
767 kfree(dest);
768 dest = NULL;
769 }
770
771 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
772 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
773 &flow_act, dest, i);
774 else
775 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
776 if (IS_ERR(rule))
777 goto err_add_rule;
778 else
779 atomic64_inc(&esw->offloads.num_flows);
780
781 kfree(dest);
782 return rule;
783
784 err_add_rule:
785 if (split)
786 mlx5_esw_vporttbl_put(esw, &fwd_attr);
787 else if (attr->chain || attr->prio)
788 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
789 err_esw_get:
790 esw_cleanup_dests(esw, attr);
791 err_create_goto_table:
792 kfree(dest);
793 return rule;
794 }
795
796 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)797 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
798 struct mlx5_flow_spec *spec,
799 struct mlx5_flow_attr *attr)
800 {
801 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
802 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
803 struct mlx5_fs_chains *chains = esw_chains(esw);
804 struct mlx5_vport_tbl_attr fwd_attr;
805 struct mlx5_flow_destination *dest;
806 struct mlx5_flow_table *fast_fdb;
807 struct mlx5_flow_table *fwd_fdb;
808 struct mlx5_flow_handle *rule;
809 int i, err = 0;
810
811 dest = kzalloc_objs(*dest, MLX5_MAX_FLOW_FWD_VPORTS + 1);
812 if (!dest)
813 return ERR_PTR(-ENOMEM);
814
815 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
816 if (IS_ERR(fast_fdb)) {
817 rule = ERR_CAST(fast_fdb);
818 goto err_get_fast;
819 }
820
821 fwd_attr.chain = attr->chain;
822 fwd_attr.prio = attr->prio;
823 fwd_attr.vport = esw_attr->in_rep->vport;
824 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
825 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
826 if (IS_ERR(fwd_fdb)) {
827 rule = ERR_CAST(fwd_fdb);
828 goto err_get_fwd;
829 }
830
831 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
832 for (i = 0; i < esw_attr->split_count; i++) {
833 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
834 /* Source port rewrite (forward to ovs internal port or statck device) isn't
835 * supported in the rule of split action.
836 */
837 err = -EOPNOTSUPP;
838 else
839 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
840
841 if (err) {
842 rule = ERR_PTR(err);
843 goto err_chain_src_rewrite;
844 }
845 }
846 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
847 dest[i].ft = fwd_fdb;
848 i++;
849
850 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
851 esw_attr->in_mdev->priv.eswitch,
852 esw_attr->in_rep->vport);
853
854 if (attr->outer_match_level != MLX5_MATCH_NONE)
855 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
856
857 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
858 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
859
860 if (IS_ERR(rule)) {
861 i = esw_attr->split_count;
862 goto err_chain_src_rewrite;
863 }
864
865 atomic64_inc(&esw->offloads.num_flows);
866
867 kfree(dest);
868 return rule;
869 err_chain_src_rewrite:
870 mlx5_esw_vporttbl_put(esw, &fwd_attr);
871 err_get_fwd:
872 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
873 err_get_fast:
874 kfree(dest);
875 return rule;
876 }
877
878 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)879 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
880 struct mlx5_flow_handle *rule,
881 struct mlx5_flow_attr *attr,
882 bool fwd_rule)
883 {
884 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
885 struct mlx5_fs_chains *chains = esw_chains(esw);
886 bool split = (esw_attr->split_count > 0);
887 struct mlx5_vport_tbl_attr fwd_attr;
888 int i;
889
890 mlx5_del_flow_rules(rule);
891
892 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
893 /* unref the term table */
894 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
895 if (esw_attr->dests[i].termtbl)
896 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
897 }
898 }
899
900 atomic64_dec(&esw->offloads.num_flows);
901
902 if (fwd_rule || split) {
903 fwd_attr.chain = attr->chain;
904 fwd_attr.prio = attr->prio;
905 fwd_attr.vport = esw_attr->in_rep->vport;
906 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
907 }
908
909 if (fwd_rule) {
910 mlx5_esw_vporttbl_put(esw, &fwd_attr);
911 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
912 } else {
913 if (split)
914 mlx5_esw_vporttbl_put(esw, &fwd_attr);
915 else if (attr->chain || attr->prio)
916 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
917 esw_cleanup_dests(esw, attr);
918 }
919 }
920
921 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)922 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
923 struct mlx5_flow_handle *rule,
924 struct mlx5_flow_attr *attr)
925 {
926 __mlx5_eswitch_del_rule(esw, rule, attr, false);
927 }
928
929 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)930 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
931 struct mlx5_flow_handle *rule,
932 struct mlx5_flow_attr *attr)
933 {
934 __mlx5_eswitch_del_rule(esw, rule, attr, true);
935 }
936
937 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)938 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
939 struct mlx5_eswitch *from_esw,
940 struct mlx5_eswitch_rep *rep,
941 u32 sqn)
942 {
943 struct mlx5_flow_act flow_act = {0};
944 struct mlx5_flow_destination dest = {};
945 struct mlx5_flow_handle *flow_rule;
946 struct mlx5_flow_spec *spec;
947 void *misc;
948 u16 vport;
949
950 spec = kvzalloc_obj(*spec);
951 if (!spec) {
952 flow_rule = ERR_PTR(-ENOMEM);
953 goto out;
954 }
955
956 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
957 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
958
959 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
960 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
961
962 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
963
964 /* source vport is the esw manager */
965 vport = from_esw->manager_vport;
966
967 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) {
968 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
969 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
970 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
971
972 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
973 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
974 mlx5_eswitch_get_vport_metadata_mask());
975
976 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
977 } else {
978 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
979 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
980
981 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
982 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
983 MLX5_CAP_GEN(from_esw->dev, vhca_id));
984
985 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
986 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
987
988 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
989 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
990 source_eswitch_owner_vhca_id);
991
992 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
993 }
994
995 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
996 dest.vport.num = rep->vport;
997 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
998 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
999 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1000
1001 if (rep->vport == MLX5_VPORT_UPLINK &&
1002 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
1003 dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
1004 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
1005 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1006 } else {
1007 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1008 dest.vport.num = rep->vport;
1009 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1010 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1011 }
1012
1013 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1014 rep->vport == MLX5_VPORT_UPLINK)
1015 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1016
1017 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw),
1018 spec, &flow_act, &dest, 1);
1019 if (IS_ERR(flow_rule))
1020 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %pe\n",
1021 flow_rule);
1022 out:
1023 kvfree(spec);
1024 return flow_rule;
1025 }
1026 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1027
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1028 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1029 {
1030 mlx5_del_flow_rules(rule);
1031 }
1032
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1033 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1034 {
1035 if (rule)
1036 mlx5_del_flow_rules(rule);
1037 }
1038
1039 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1040 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1041 {
1042 struct mlx5_flow_destination dest = {};
1043 struct mlx5_flow_act flow_act = {0};
1044 struct mlx5_flow_handle *flow_rule;
1045 struct mlx5_flow_spec *spec;
1046
1047 spec = kvzalloc_obj(*spec);
1048 if (!spec)
1049 return ERR_PTR(-ENOMEM);
1050
1051 MLX5_SET(fte_match_param, spec->match_criteria,
1052 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1053 MLX5_SET(fte_match_param, spec->match_criteria,
1054 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1055 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1056 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1057
1058 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1059 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1060 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1061
1062 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1063 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1064 dest.vport.num = vport_num;
1065
1066 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1067 spec, &flow_act, &dest, 1);
1068 if (IS_ERR(flow_rule))
1069 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %pe\n",
1070 vport_num, flow_rule);
1071
1072 kvfree(spec);
1073 return flow_rule;
1074 }
1075
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1076 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1077 {
1078 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1079 MLX5_FDB_TO_VPORT_REG_C_1;
1080 }
1081
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1082 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1083 {
1084 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1085 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1086 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1087 u8 curr, wanted;
1088 int err;
1089
1090 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1091 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1092 return 0;
1093
1094 MLX5_SET(query_esw_vport_context_in, in, opcode,
1095 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1096 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1097 if (err)
1098 return err;
1099
1100 curr = MLX5_GET(query_esw_vport_context_out, out,
1101 esw_vport_context.fdb_to_vport_reg_c_id);
1102 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1103 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1104 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1105
1106 if (enable)
1107 curr |= wanted;
1108 else
1109 curr &= ~wanted;
1110
1111 MLX5_SET(modify_esw_vport_context_in, min,
1112 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1113 MLX5_SET(modify_esw_vport_context_in, min,
1114 field_select.fdb_to_vport_reg_c_id, 1);
1115
1116 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1117 if (!err) {
1118 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1119 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1120 else
1121 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1122 }
1123
1124 return err;
1125 }
1126
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1127 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1128 struct mlx5_core_dev *peer_dev,
1129 struct mlx5_flow_spec *spec,
1130 struct mlx5_flow_destination *dest)
1131 {
1132 void *misc;
1133
1134 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1135 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1136 misc_parameters_2);
1137 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1138 mlx5_eswitch_get_vport_metadata_mask());
1139
1140 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1141 } else {
1142 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1143 misc_parameters);
1144
1145 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1146 MLX5_CAP_GEN(peer_dev, vhca_id));
1147
1148 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1149
1150 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1151 misc_parameters);
1152 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1153 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1154 source_eswitch_owner_vhca_id);
1155 }
1156
1157 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1158 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1159 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1160 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1161 }
1162
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1163 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1164 struct mlx5_eswitch *peer_esw,
1165 struct mlx5_flow_spec *spec,
1166 u16 vport)
1167 {
1168 void *misc;
1169
1170 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1171 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1172 misc_parameters_2);
1173 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1174 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1175 vport));
1176 } else {
1177 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1178 misc_parameters);
1179 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1180 }
1181 }
1182
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1183 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1184 struct mlx5_core_dev *peer_dev)
1185 {
1186 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch;
1187 struct mlx5_flow_destination dest = {};
1188 struct mlx5_flow_act flow_act = {0};
1189 struct mlx5_flow_handle **flows;
1190 struct mlx5_flow_handle *flow;
1191 struct mlx5_vport *peer_vport;
1192 struct mlx5_flow_spec *spec;
1193 int err;
1194 unsigned long i;
1195 void *misc;
1196
1197 if (!MLX5_VPORT_MANAGER(peer_dev) &&
1198 !mlx5_core_is_ecpf_esw_manager(peer_dev))
1199 return 0;
1200
1201 spec = kvzalloc_obj(*spec);
1202 if (!spec)
1203 return -ENOMEM;
1204
1205 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1206
1207 flows = kvzalloc_objs(*flows, peer_esw->total_vports);
1208 if (!flows) {
1209 err = -ENOMEM;
1210 goto alloc_flows_err;
1211 }
1212
1213 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1214 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1215 misc_parameters);
1216
1217 if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
1218 mlx5_esw_host_functions_enabled(peer_dev)) {
1219 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
1220 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec,
1221 MLX5_VPORT_PF);
1222
1223 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1224 spec, &flow_act, &dest, 1);
1225 if (IS_ERR(flow)) {
1226 err = PTR_ERR(flow);
1227 goto add_pf_flow_err;
1228 }
1229 flows[peer_vport->index] = flow;
1230 }
1231
1232 if (mlx5_ecpf_vport_exists(peer_dev)) {
1233 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF);
1234 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1235 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1236 spec, &flow_act, &dest, 1);
1237 if (IS_ERR(flow)) {
1238 err = PTR_ERR(flow);
1239 goto add_ecpf_flow_err;
1240 }
1241 flows[peer_vport->index] = flow;
1242 }
1243
1244 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1245 mlx5_core_max_vfs(peer_dev)) {
1246 esw_set_peer_miss_rule_source_port(esw, peer_esw, spec,
1247 peer_vport->vport);
1248 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1249 spec, &flow_act, &dest, 1);
1250 if (IS_ERR(flow)) {
1251 err = PTR_ERR(flow);
1252 goto add_vf_flow_err;
1253 }
1254 flows[peer_vport->index] = flow;
1255 }
1256
1257 if (mlx5_core_ec_sriov_enabled(peer_dev)) {
1258 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1259 mlx5_core_max_ec_vfs(peer_dev)) {
1260 esw_set_peer_miss_rule_source_port(esw, peer_esw,
1261 spec,
1262 peer_vport->vport);
1263 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1264 spec, &flow_act, &dest, 1);
1265 if (IS_ERR(flow)) {
1266 err = PTR_ERR(flow);
1267 goto add_ec_vf_flow_err;
1268 }
1269 flows[peer_vport->index] = flow;
1270 }
1271 }
1272
1273 err = xa_insert(&esw->fdb_table.offloads.peer_miss_rules,
1274 MLX5_CAP_GEN(peer_dev, vhca_id), flows, GFP_KERNEL);
1275 if (err)
1276 goto add_ec_vf_flow_err;
1277
1278 kvfree(spec);
1279 return 0;
1280
1281 add_ec_vf_flow_err:
1282 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1283 mlx5_core_max_ec_vfs(peer_dev)) {
1284 if (!flows[peer_vport->index])
1285 continue;
1286 mlx5_del_flow_rules(flows[peer_vport->index]);
1287 }
1288 add_vf_flow_err:
1289 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1290 mlx5_core_max_vfs(peer_dev)) {
1291 if (!flows[peer_vport->index])
1292 continue;
1293 mlx5_del_flow_rules(flows[peer_vport->index]);
1294 }
1295 if (mlx5_ecpf_vport_exists(peer_dev)) {
1296 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF);
1297 mlx5_del_flow_rules(flows[peer_vport->index]);
1298 }
1299 add_ecpf_flow_err:
1300
1301 if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
1302 mlx5_esw_host_functions_enabled(peer_dev)) {
1303 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
1304 mlx5_del_flow_rules(flows[peer_vport->index]);
1305 }
1306 add_pf_flow_err:
1307 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1308 kvfree(flows);
1309 alloc_flows_err:
1310 kvfree(spec);
1311 return err;
1312 }
1313
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1314 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1315 struct mlx5_core_dev *peer_dev)
1316 {
1317 struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch;
1318 u16 peer_vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1319 struct mlx5_flow_handle **flows;
1320 struct mlx5_vport *peer_vport;
1321 unsigned long i;
1322
1323 flows = xa_erase(&esw->fdb_table.offloads.peer_miss_rules,
1324 peer_vhca_id);
1325 if (!flows)
1326 return;
1327
1328 if (mlx5_core_ec_sriov_enabled(peer_dev)) {
1329 mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1330 mlx5_core_max_ec_vfs(peer_dev))
1331 mlx5_del_flow_rules(flows[peer_vport->index]);
1332 }
1333
1334 mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1335 mlx5_core_max_vfs(peer_dev))
1336 mlx5_del_flow_rules(flows[peer_vport->index]);
1337
1338 if (mlx5_ecpf_vport_exists(peer_dev)) {
1339 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_ECPF);
1340 mlx5_del_flow_rules(flows[peer_vport->index]);
1341 }
1342
1343 if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
1344 mlx5_esw_host_functions_enabled(peer_dev)) {
1345 peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
1346 mlx5_del_flow_rules(flows[peer_vport->index]);
1347 }
1348
1349 kvfree(flows);
1350 }
1351
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1352 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1353 {
1354 struct mlx5_flow_act flow_act = {0};
1355 struct mlx5_flow_destination dest = {};
1356 struct mlx5_flow_handle *flow_rule = NULL;
1357 struct mlx5_flow_spec *spec;
1358 void *headers_c;
1359 void *headers_v;
1360 int err = 0;
1361 u8 *dmac_c;
1362 u8 *dmac_v;
1363
1364 spec = kvzalloc_obj(*spec);
1365 if (!spec) {
1366 err = -ENOMEM;
1367 goto out;
1368 }
1369
1370 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1371 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1372 outer_headers);
1373 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1374 outer_headers.dmac_47_16);
1375 dmac_c[0] = 0x01;
1376
1377 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1378 dest.vport.num = esw->manager_vport;
1379 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1380
1381 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1382 spec, &flow_act, &dest, 1);
1383 if (IS_ERR(flow_rule)) {
1384 err = PTR_ERR(flow_rule);
1385 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1386 goto out;
1387 }
1388
1389 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1390
1391 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1392 outer_headers);
1393 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1394 outer_headers.dmac_47_16);
1395 dmac_v[0] = 0x01;
1396 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1397 spec, &flow_act, &dest, 1);
1398 if (IS_ERR(flow_rule)) {
1399 err = PTR_ERR(flow_rule);
1400 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1401 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1402 goto out;
1403 }
1404
1405 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1406
1407 out:
1408 kvfree(spec);
1409 return err;
1410 }
1411
1412 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1413 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1414 {
1415 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1416 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1417 struct mlx5_flow_context *flow_context;
1418 struct mlx5_flow_handle *flow_rule;
1419 struct mlx5_flow_destination dest;
1420 struct mlx5_flow_spec *spec;
1421 void *misc;
1422
1423 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1424 return ERR_PTR(-EOPNOTSUPP);
1425
1426 spec = kvzalloc_obj(*spec);
1427 if (!spec)
1428 return ERR_PTR(-ENOMEM);
1429
1430 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1431 misc_parameters_2);
1432 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1433 ESW_REG_C0_USER_DATA_METADATA_MASK);
1434 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1435 misc_parameters_2);
1436 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1437 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1438 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1439 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1440 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1441
1442 flow_context = &spec->flow_context;
1443 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1444 flow_context->flow_tag = tag;
1445 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1446 dest.ft = esw->offloads.ft_offloads;
1447
1448 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1449 kvfree(spec);
1450
1451 if (IS_ERR(flow_rule))
1452 esw_warn(esw->dev,
1453 "Failed to create restore rule for tag: %d, err(%d)\n",
1454 tag, (int)PTR_ERR(flow_rule));
1455
1456 return flow_rule;
1457 }
1458
1459 struct mlx5_flow_group *
mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * ft)1460 mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw,
1461 struct mlx5_flow_table *ft)
1462 {
1463 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1464 struct mlx5_flow_group *fg;
1465 void *match_criteria;
1466 void *flow_group_in;
1467
1468 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
1469 return ERR_PTR(-EOPNOTSUPP);
1470
1471 if (IS_ERR(ft))
1472 return ERR_CAST(ft);
1473
1474 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1475 if (!flow_group_in)
1476 return ERR_PTR(-ENOMEM);
1477
1478 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1479 match_criteria);
1480 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1481 MLX5_MATCH_MISC_PARAMETERS_2);
1482 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1483 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1484 ft->max_fte - 1);
1485
1486 MLX5_SET(fte_match_param, match_criteria,
1487 misc_parameters_2.metadata_reg_c_0,
1488 mlx5_eswitch_get_vport_metadata_mask());
1489
1490 fg = mlx5_create_flow_group(ft, flow_group_in);
1491 kvfree(flow_group_in);
1492 if (IS_ERR(fg))
1493 esw_warn(esw->dev, "Can't create LAG demux flow group\n");
1494
1495 return fg;
1496 }
1497
1498 struct mlx5_flow_handle *
mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch * esw,u16 vport_num,struct mlx5_flow_table * lag_ft)1499 mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num,
1500 struct mlx5_flow_table *lag_ft)
1501 {
1502 struct mlx5_flow_spec *spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1503 struct mlx5_flow_destination dest = {};
1504 struct mlx5_flow_act flow_act = {};
1505 struct mlx5_flow_handle *ret;
1506 void *misc;
1507
1508 if (!spec)
1509 return ERR_PTR(-ENOMEM);
1510
1511 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1512 kvfree(spec);
1513 return ERR_PTR(-EOPNOTSUPP);
1514 }
1515
1516 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1517 misc_parameters_2);
1518 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1519 mlx5_eswitch_get_vport_metadata_mask());
1520 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1521
1522 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1523 misc_parameters_2);
1524 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1525 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1526
1527 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1528 dest.type = MLX5_FLOW_DESTINATION_TYPE_VHCA_RX;
1529 dest.vhca.id = MLX5_CAP_GEN(esw->dev, vhca_id);
1530
1531 ret = mlx5_add_flow_rules(lag_ft, spec, &flow_act, &dest, 1);
1532 kvfree(spec);
1533 return ret;
1534 }
1535
1536 #define MAX_PF_SQ 256
1537 #define MAX_SQ_NVPORTS 32
1538
1539 void
mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in,int match_params)1540 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1541 u32 *flow_group_in,
1542 int match_params)
1543 {
1544 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1545 flow_group_in,
1546 match_criteria);
1547
1548 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1549 MLX5_SET(create_flow_group_in, flow_group_in,
1550 match_criteria_enable,
1551 MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1552
1553 MLX5_SET(fte_match_param, match_criteria,
1554 misc_parameters_2.metadata_reg_c_0,
1555 mlx5_eswitch_get_vport_metadata_mask());
1556 } else {
1557 MLX5_SET(create_flow_group_in, flow_group_in,
1558 match_criteria_enable,
1559 MLX5_MATCH_MISC_PARAMETERS | match_params);
1560
1561 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1562 misc_parameters.source_port);
1563 }
1564 }
1565
1566 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1567 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1568 {
1569 struct mlx5_vport_tbl_attr attr;
1570 struct mlx5_vport *vport;
1571 unsigned long i;
1572
1573 attr.chain = 0;
1574 attr.prio = 1;
1575 mlx5_esw_for_each_vport(esw, i, vport) {
1576 attr.vport = vport->vport;
1577 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1578 mlx5_esw_vporttbl_put(esw, &attr);
1579 }
1580 }
1581
esw_vport_tbl_get(struct mlx5_eswitch * esw)1582 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1583 {
1584 struct mlx5_vport_tbl_attr attr;
1585 struct mlx5_flow_table *fdb;
1586 struct mlx5_vport *vport;
1587 unsigned long i;
1588
1589 attr.chain = 0;
1590 attr.prio = 1;
1591 mlx5_esw_for_each_vport(esw, i, vport) {
1592 attr.vport = vport->vport;
1593 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1594 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1595 if (IS_ERR(fdb))
1596 goto out;
1597 }
1598 return 0;
1599
1600 out:
1601 esw_vport_tbl_put(esw);
1602 return PTR_ERR(fdb);
1603 }
1604
1605 #define fdb_modify_header_fwd_to_table_supported(esw) \
1606 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1607 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1608 {
1609 struct mlx5_core_dev *dev = esw->dev;
1610
1611 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1612 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1613
1614 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1615 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1616 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1617 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1618 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1619 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1620 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1621 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1622 /* Disabled when ttl workaround is needed, e.g
1623 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1624 */
1625 esw_warn(dev,
1626 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1627 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1628 } else {
1629 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1630 esw_info(dev, "Supported tc chains and prios offload\n");
1631 }
1632
1633 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1634 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1635 }
1636
1637 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1638 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1639 {
1640 struct mlx5_core_dev *dev = esw->dev;
1641 struct mlx5_flow_table *nf_ft, *ft;
1642 struct mlx5_chains_attr attr = {};
1643 struct mlx5_fs_chains *chains;
1644 int err;
1645
1646 esw_init_chains_offload_flags(esw, &attr.flags);
1647 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1648 attr.max_grp_num = esw->params.large_group_num;
1649 attr.default_ft = miss_fdb;
1650 attr.mapping = esw->offloads.reg_c0_obj_pool;
1651 attr.fs_base_prio = FDB_BYPASS_PATH;
1652
1653 chains = mlx5_chains_create(dev, &attr);
1654 if (IS_ERR(chains)) {
1655 err = PTR_ERR(chains);
1656 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1657 return err;
1658 }
1659 mlx5_chains_print_info(chains);
1660
1661 esw->fdb_table.offloads.esw_chains_priv = chains;
1662
1663 /* Create tc_end_ft which is the always created ft chain */
1664 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1665 1, 0);
1666 if (IS_ERR(nf_ft)) {
1667 err = PTR_ERR(nf_ft);
1668 goto nf_ft_err;
1669 }
1670
1671 /* Always open the root for fast path */
1672 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1673 if (IS_ERR(ft)) {
1674 err = PTR_ERR(ft);
1675 goto level_0_err;
1676 }
1677
1678 /* Open level 1 for split fdb rules now if prios isn't supported */
1679 if (!mlx5_chains_prios_supported(chains)) {
1680 err = esw_vport_tbl_get(esw);
1681 if (err)
1682 goto level_1_err;
1683 }
1684
1685 mlx5_chains_set_end_ft(chains, nf_ft);
1686
1687 return 0;
1688
1689 level_1_err:
1690 mlx5_chains_put_table(chains, 0, 1, 0);
1691 level_0_err:
1692 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1693 nf_ft_err:
1694 mlx5_chains_destroy(chains);
1695 esw->fdb_table.offloads.esw_chains_priv = NULL;
1696
1697 return err;
1698 }
1699
1700 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1701 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1702 {
1703 if (!mlx5_chains_prios_supported(chains))
1704 esw_vport_tbl_put(esw);
1705 mlx5_chains_put_table(chains, 0, 1, 0);
1706 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1707 mlx5_chains_destroy(chains);
1708 }
1709
1710 #else /* CONFIG_MLX5_CLS_ACT */
1711
1712 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1713 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1714 { return 0; }
1715
1716 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1717 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1718 {}
1719
1720 #endif
1721
1722 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1723 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1724 struct mlx5_flow_table *fdb,
1725 u32 *flow_group_in,
1726 int *ix)
1727 {
1728 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1729 struct mlx5_flow_group *g;
1730 void *match_criteria;
1731 int count, err = 0;
1732
1733 memset(flow_group_in, 0, inlen);
1734
1735 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS);
1736
1737 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1738 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1739
1740 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1741 MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1742 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1743 misc_parameters.source_eswitch_owner_vhca_id);
1744 MLX5_SET(create_flow_group_in, flow_group_in,
1745 source_eswitch_owner_vhca_id_valid, 1);
1746 }
1747
1748 /* See comment at table_size calculation */
1749 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1750 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1751 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1752 *ix += count;
1753
1754 g = mlx5_create_flow_group(fdb, flow_group_in);
1755 if (IS_ERR(g)) {
1756 err = PTR_ERR(g);
1757 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1758 goto out;
1759 }
1760 esw->fdb_table.offloads.send_to_vport_grp = g;
1761
1762 out:
1763 return err;
1764 }
1765
1766 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1767 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1768 struct mlx5_flow_table *fdb,
1769 u32 *flow_group_in,
1770 int *ix)
1771 {
1772 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1773 struct mlx5_flow_group *g;
1774 void *match_criteria;
1775 int err = 0;
1776
1777 if (!esw_src_port_rewrite_supported(esw))
1778 return 0;
1779
1780 memset(flow_group_in, 0, inlen);
1781
1782 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1783 MLX5_MATCH_MISC_PARAMETERS_2);
1784
1785 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1786
1787 MLX5_SET(fte_match_param, match_criteria,
1788 misc_parameters_2.metadata_reg_c_0,
1789 mlx5_eswitch_get_vport_metadata_mask());
1790 MLX5_SET(fte_match_param, match_criteria,
1791 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1792
1793 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1794 MLX5_SET(create_flow_group_in, flow_group_in,
1795 end_flow_index, *ix + esw->total_vports - 1);
1796 *ix += esw->total_vports;
1797
1798 g = mlx5_create_flow_group(fdb, flow_group_in);
1799 if (IS_ERR(g)) {
1800 err = PTR_ERR(g);
1801 esw_warn(esw->dev,
1802 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1803 goto send_vport_meta_err;
1804 }
1805 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1806
1807 return 0;
1808
1809 send_vport_meta_err:
1810 return err;
1811 }
1812
1813 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1814 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1815 struct mlx5_flow_table *fdb,
1816 u32 *flow_group_in,
1817 int *ix)
1818 {
1819 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1820 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1821 struct mlx5_flow_group *g;
1822 void *match_criteria;
1823 int err = 0;
1824
1825 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1826 return 0;
1827
1828 memset(flow_group_in, 0, inlen);
1829
1830 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
1831
1832 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1833 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1834 flow_group_in,
1835 match_criteria);
1836
1837 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1838 misc_parameters.source_eswitch_owner_vhca_id);
1839
1840 MLX5_SET(create_flow_group_in, flow_group_in,
1841 source_eswitch_owner_vhca_id_valid, 1);
1842 }
1843
1844 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1845 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1846 *ix + max_peer_ports);
1847 *ix += max_peer_ports + 1;
1848
1849 g = mlx5_create_flow_group(fdb, flow_group_in);
1850 if (IS_ERR(g)) {
1851 err = PTR_ERR(g);
1852 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1853 goto out;
1854 }
1855 esw->fdb_table.offloads.peer_miss_grp = g;
1856
1857 out:
1858 return err;
1859 }
1860
1861 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1862 esw_create_miss_group(struct mlx5_eswitch *esw,
1863 struct mlx5_flow_table *fdb,
1864 u32 *flow_group_in,
1865 int *ix)
1866 {
1867 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1868 struct mlx5_flow_group *g;
1869 void *match_criteria;
1870 int err = 0;
1871 u8 *dmac;
1872
1873 memset(flow_group_in, 0, inlen);
1874
1875 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1876 MLX5_MATCH_OUTER_HEADERS);
1877 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1878 match_criteria);
1879 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1880 outer_headers.dmac_47_16);
1881 dmac[0] = 0x01;
1882
1883 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1884 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1885 *ix + MLX5_ESW_MISS_FLOWS);
1886
1887 g = mlx5_create_flow_group(fdb, flow_group_in);
1888 if (IS_ERR(g)) {
1889 err = PTR_ERR(g);
1890 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1891 goto miss_err;
1892 }
1893 esw->fdb_table.offloads.miss_grp = g;
1894
1895 err = esw_add_fdb_miss_rule(esw);
1896 if (err)
1897 goto miss_rule_err;
1898
1899 return 0;
1900
1901 miss_rule_err:
1902 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1903 miss_err:
1904 return err;
1905 }
1906
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1907 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1908 {
1909 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1910 struct mlx5_flow_table_attr ft_attr = {};
1911 struct mlx5_core_dev *dev = esw->dev;
1912 struct mlx5_flow_namespace *root_ns;
1913 struct mlx5_flow_table *fdb = NULL;
1914 int table_size, ix = 0, err = 0;
1915 u32 flags = 0, *flow_group_in;
1916
1917 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1918
1919 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1920 if (!flow_group_in)
1921 return -ENOMEM;
1922
1923 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1924 if (!root_ns) {
1925 esw_warn(dev, "Failed to get FDB flow namespace\n");
1926 err = -EOPNOTSUPP;
1927 goto ns_err;
1928 }
1929 esw->fdb_table.offloads.ns = root_ns;
1930 err = mlx5_flow_namespace_set_mode(root_ns,
1931 esw->dev->priv.steering->mode);
1932 if (err) {
1933 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1934 goto ns_err;
1935 }
1936
1937 /* To be strictly correct:
1938 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1939 * should be:
1940 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1941 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1942 * but as the peer device might not be in switchdev mode it's not
1943 * possible. We use the fact that by default FW sets max vfs and max sfs
1944 * to the same value on both devices. If it needs to be changed in the future note
1945 * the peer miss group should also be created based on the number of
1946 * total vports of the peer (currently is also uses esw->total_vports).
1947 */
1948 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1949 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1950
1951 /* create the slow path fdb with encap set, so further table instances
1952 * can be created at run time while VFs are probed if the FW allows that.
1953 */
1954 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1955 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1956 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1957
1958 ft_attr.flags = flags;
1959 ft_attr.max_fte = table_size;
1960 ft_attr.prio = FDB_SLOW_PATH;
1961
1962 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1963 if (IS_ERR(fdb)) {
1964 err = PTR_ERR(fdb);
1965 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1966 goto slow_fdb_err;
1967 }
1968 esw->fdb_table.offloads.slow_fdb = fdb;
1969
1970 /* Create empty TC-miss managed table. This allows plugging in following
1971 * priorities without directly exposing their level 0 table to
1972 * eswitch_offloads and passing it as miss_fdb to following call to
1973 * esw_chains_create().
1974 */
1975 memset(&ft_attr, 0, sizeof(ft_attr));
1976 ft_attr.prio = FDB_TC_MISS;
1977 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1978 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1979 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1980 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1981 goto tc_miss_table_err;
1982 }
1983
1984 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1985 if (err) {
1986 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1987 goto fdb_chains_err;
1988 }
1989
1990 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1991 if (err)
1992 goto send_vport_err;
1993
1994 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1995 if (err)
1996 goto send_vport_meta_err;
1997
1998 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1999 if (err)
2000 goto peer_miss_err;
2001
2002 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
2003 if (err)
2004 goto miss_err;
2005
2006 kvfree(flow_group_in);
2007 return 0;
2008
2009 miss_err:
2010 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
2011 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
2012 peer_miss_err:
2013 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
2014 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
2015 send_vport_meta_err:
2016 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
2017 send_vport_err:
2018 esw_chains_destroy(esw, esw_chains(esw));
2019 fdb_chains_err:
2020 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
2021 tc_miss_table_err:
2022 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
2023 slow_fdb_err:
2024 /* Holds true only as long as DMFS is the default */
2025 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
2026 ns_err:
2027 kvfree(flow_group_in);
2028 return err;
2029 }
2030
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)2031 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
2032 {
2033 if (!mlx5_eswitch_get_slow_fdb(esw))
2034 return;
2035
2036 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
2037 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
2038 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
2039 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
2040 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
2041 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
2042 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
2043 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
2044 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
2045
2046 esw_chains_destroy(esw, esw_chains(esw));
2047
2048 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
2049 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
2050 /* Holds true only as long as DMFS is the default */
2051 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
2052 MLX5_FLOW_STEERING_MODE_DMFS);
2053 }
2054
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)2055 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
2056 {
2057 int nvports;
2058
2059 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
2060 if (mlx5e_tc_int_port_supported(esw))
2061 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
2062
2063 return nvports;
2064 }
2065
esw_create_offloads_table(struct mlx5_eswitch * esw)2066 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
2067 {
2068 struct mlx5_flow_table_attr ft_attr = {};
2069 struct mlx5_core_dev *dev = esw->dev;
2070 struct mlx5_flow_table *ft_offloads;
2071 struct mlx5_flow_namespace *ns;
2072 int err = 0;
2073
2074 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2075 if (!ns) {
2076 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2077 return -EOPNOTSUPP;
2078 }
2079
2080 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
2081 MLX5_ESW_FT_OFFLOADS_DROP_RULE;
2082 ft_attr.prio = 1;
2083
2084 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
2085 if (IS_ERR(ft_offloads)) {
2086 err = PTR_ERR(ft_offloads);
2087 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
2088 return err;
2089 }
2090
2091 esw->offloads.ft_offloads = ft_offloads;
2092 return 0;
2093 }
2094
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2095 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2096 {
2097 struct mlx5_esw_offload *offloads = &esw->offloads;
2098
2099 mlx5_destroy_flow_table(offloads->ft_offloads);
2100 }
2101
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2102 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2103 {
2104 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2105 struct mlx5_flow_group *g;
2106 u32 *flow_group_in;
2107 int nvports;
2108 int err = 0;
2109
2110 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2111 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2112 if (!flow_group_in)
2113 return -ENOMEM;
2114
2115 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
2116
2117 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2118 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2119
2120 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2121
2122 if (IS_ERR(g)) {
2123 err = PTR_ERR(g);
2124 esw_warn(esw->dev, "Failed to create vport rx group err %d\n",
2125 err);
2126 goto out;
2127 }
2128
2129 esw->offloads.vport_rx_group = g;
2130 out:
2131 kvfree(flow_group_in);
2132 return err;
2133 }
2134
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2135 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2136 {
2137 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2138 }
2139
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2140 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2141 {
2142 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2143 * for the drop rule, which is placed at the end of the table.
2144 * So return the total of vport and int_port as rule index.
2145 */
2146 return esw_get_nr_ft_offloads_steering_src_ports(esw);
2147 }
2148
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2149 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2150 {
2151 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2152 struct mlx5_flow_group *g;
2153 u32 *flow_group_in;
2154 int flow_index;
2155 int err = 0;
2156
2157 flow_index = esw_create_vport_rx_drop_rule_index(esw);
2158
2159 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2160 if (!flow_group_in)
2161 return -ENOMEM;
2162
2163 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2164 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2165
2166 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2167
2168 if (IS_ERR(g)) {
2169 err = PTR_ERR(g);
2170 esw_warn(esw->dev,
2171 "Failed to create vport rx drop group err %d\n", err);
2172 goto out;
2173 }
2174
2175 esw->offloads.vport_rx_drop_group = g;
2176 out:
2177 kvfree(flow_group_in);
2178 return err;
2179 }
2180
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2181 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2182 {
2183 if (esw->offloads.vport_rx_drop_group)
2184 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2185 }
2186
2187 void
mlx5_esw_set_spec_source_port(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_spec * spec)2188 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2189 u16 vport,
2190 struct mlx5_flow_spec *spec)
2191 {
2192 void *misc;
2193
2194 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2195 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2196 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2197 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2198
2199 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2200 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2201 mlx5_eswitch_get_vport_metadata_mask());
2202
2203 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2204 } else {
2205 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2206 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2207
2208 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2209 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2210
2211 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2212 }
2213 }
2214
2215 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2216 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2217 struct mlx5_flow_destination *dest)
2218 {
2219 struct mlx5_flow_act flow_act = {0};
2220 struct mlx5_flow_handle *flow_rule;
2221 struct mlx5_flow_spec *spec;
2222
2223 spec = kvzalloc_obj(*spec);
2224 if (!spec) {
2225 flow_rule = ERR_PTR(-ENOMEM);
2226 goto out;
2227 }
2228
2229 mlx5_esw_set_spec_source_port(esw, vport, spec);
2230
2231 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2232 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2233 &flow_act, dest, 1);
2234 if (IS_ERR(flow_rule)) {
2235 esw_warn(esw->dev,
2236 "fs offloads: Failed to add vport rx rule err %pe\n",
2237 flow_rule);
2238 goto out;
2239 }
2240
2241 out:
2242 kvfree(spec);
2243 return flow_rule;
2244 }
2245
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2246 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2247 {
2248 struct mlx5_flow_act flow_act = {};
2249 struct mlx5_flow_handle *flow_rule;
2250
2251 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2252 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2253 &flow_act, NULL, 0);
2254 if (IS_ERR(flow_rule)) {
2255 esw_warn(esw->dev,
2256 "fs offloads: Failed to add vport rx drop rule err %pe\n",
2257 flow_rule);
2258 return PTR_ERR(flow_rule);
2259 }
2260
2261 esw->offloads.vport_rx_drop_rule = flow_rule;
2262
2263 return 0;
2264 }
2265
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2266 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2267 {
2268 if (esw->offloads.vport_rx_drop_rule)
2269 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2270 }
2271
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2272 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2273 {
2274 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2275 struct mlx5_core_dev *dev = esw->dev;
2276 struct mlx5_vport *vport;
2277 unsigned long i;
2278
2279 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2280 return -EOPNOTSUPP;
2281
2282 if (!mlx5_esw_is_fdb_created(esw))
2283 return -EOPNOTSUPP;
2284
2285 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2286 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2287 mlx5_mode = MLX5_INLINE_MODE_NONE;
2288 goto out;
2289 case MLX5_CAP_INLINE_MODE_L2:
2290 mlx5_mode = MLX5_INLINE_MODE_L2;
2291 goto out;
2292 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2293 goto query_vports;
2294 }
2295
2296 query_vports:
2297 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2298 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2299 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2300 if (prev_mlx5_mode != mlx5_mode)
2301 return -EINVAL;
2302 prev_mlx5_mode = mlx5_mode;
2303 }
2304
2305 out:
2306 *mode = mlx5_mode;
2307 return 0;
2308 }
2309
esw_destroy_restore_table(struct mlx5_eswitch * esw)2310 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2311 {
2312 struct mlx5_esw_offload *offloads = &esw->offloads;
2313
2314 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2315 return;
2316
2317 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2318 mlx5_destroy_flow_group(offloads->restore_group);
2319 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2320 }
2321
esw_create_restore_table(struct mlx5_eswitch * esw)2322 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2323 {
2324 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2325 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2326 struct mlx5_flow_table_attr ft_attr = {};
2327 struct mlx5_core_dev *dev = esw->dev;
2328 struct mlx5_flow_namespace *ns;
2329 struct mlx5_modify_hdr *mod_hdr;
2330 void *match_criteria, *misc;
2331 struct mlx5_flow_table *ft;
2332 struct mlx5_flow_group *g;
2333 u32 *flow_group_in;
2334 int err = 0;
2335
2336 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2337 return 0;
2338
2339 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2340 if (!ns) {
2341 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2342 return -EOPNOTSUPP;
2343 }
2344
2345 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2346 if (!flow_group_in) {
2347 err = -ENOMEM;
2348 goto out_free;
2349 }
2350
2351 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2352 ft = mlx5_create_flow_table(ns, &ft_attr);
2353 if (IS_ERR(ft)) {
2354 err = PTR_ERR(ft);
2355 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2356 err);
2357 goto out_free;
2358 }
2359
2360 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2361 match_criteria);
2362 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2363 misc_parameters_2);
2364
2365 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2366 ESW_REG_C0_USER_DATA_METADATA_MASK);
2367 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2368 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2369 ft_attr.max_fte - 1);
2370 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2371 MLX5_MATCH_MISC_PARAMETERS_2);
2372 g = mlx5_create_flow_group(ft, flow_group_in);
2373 if (IS_ERR(g)) {
2374 err = PTR_ERR(g);
2375 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2376 err);
2377 goto err_group;
2378 }
2379
2380 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2381 MLX5_SET(copy_action_in, modact, src_field,
2382 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2383 MLX5_SET(copy_action_in, modact, dst_field,
2384 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2385 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2386 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2387 modact);
2388 if (IS_ERR(mod_hdr)) {
2389 err = PTR_ERR(mod_hdr);
2390 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2391 err);
2392 goto err_mod_hdr;
2393 }
2394
2395 esw->offloads.ft_offloads_restore = ft;
2396 esw->offloads.restore_group = g;
2397 esw->offloads.restore_copy_hdr_id = mod_hdr;
2398
2399 kvfree(flow_group_in);
2400
2401 return 0;
2402
2403 err_mod_hdr:
2404 mlx5_destroy_flow_group(g);
2405 err_group:
2406 mlx5_destroy_flow_table(ft);
2407 out_free:
2408 kvfree(flow_group_in);
2409
2410 return err;
2411 }
2412
esw_mode_change(struct mlx5_eswitch * esw,u16 mode)2413 static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode)
2414 {
2415 mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp);
2416 if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV ||
2417 mlx5_core_mp_enabled(esw->dev)) {
2418 esw->mode = mode;
2419 mlx5_rescan_drivers_locked(esw->dev);
2420 mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp);
2421 return;
2422 }
2423
2424 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
2425 mlx5_rescan_drivers_locked(esw->dev);
2426 esw->mode = mode;
2427 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
2428 mlx5_rescan_drivers_locked(esw->dev);
2429 mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp);
2430 }
2431
mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch * esw)2432 static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw)
2433 {
2434 if (!esw->fdb_table.offloads.drop_root)
2435 return;
2436
2437 esw_debug(esw->dev, "Destroying FDB drop root table %#x fc %#x\n",
2438 esw->fdb_table.offloads.drop_root->id,
2439 esw->fdb_table.offloads.drop_root_fc->id);
2440 mlx5_del_flow_rules(esw->fdb_table.offloads.drop_root_rule);
2441 /* Don't free flow counter here, can be reused on a later activation */
2442 mlx5_destroy_flow_table(esw->fdb_table.offloads.drop_root);
2443 esw->fdb_table.offloads.drop_root_rule = NULL;
2444 esw->fdb_table.offloads.drop_root = NULL;
2445 }
2446
mlx5_esw_fdb_drop_create(struct mlx5_eswitch * esw)2447 static int mlx5_esw_fdb_drop_create(struct mlx5_eswitch *esw)
2448 {
2449 struct mlx5_flow_destination drop_fc_dst = {};
2450 struct mlx5_flow_table_attr ft_attr = {};
2451 struct mlx5_flow_destination *dst = NULL;
2452 struct mlx5_core_dev *dev = esw->dev;
2453 struct mlx5_flow_namespace *root_ns;
2454 struct mlx5_flow_act flow_act = {};
2455 struct mlx5_flow_handle *flow_rule;
2456 struct mlx5_flow_table *table;
2457 int err = 0, dst_num = 0;
2458
2459 if (esw->fdb_table.offloads.drop_root)
2460 return 0;
2461
2462 root_ns = esw->fdb_table.offloads.ns;
2463
2464 ft_attr.prio = FDB_DROP_ROOT;
2465 ft_attr.max_fte = 1;
2466 ft_attr.autogroup.max_num_groups = 1;
2467 table = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
2468 if (IS_ERR(table)) {
2469 esw_warn(dev, "Failed to create fdb drop root table, err %pe\n",
2470 table);
2471 return PTR_ERR(table);
2472 }
2473
2474 /* Drop FC reusable, create once on first deactivation of FDB */
2475 if (!esw->fdb_table.offloads.drop_root_fc) {
2476 struct mlx5_fc *counter = mlx5_fc_create(dev, 0);
2477
2478 err = PTR_ERR_OR_ZERO(counter);
2479 if (err)
2480 esw_warn(esw->dev, "create fdb drop fc err %d\n", err);
2481 else
2482 esw->fdb_table.offloads.drop_root_fc = counter;
2483 }
2484
2485 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2486
2487 if (esw->fdb_table.offloads.drop_root_fc) {
2488 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2489 drop_fc_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2490 drop_fc_dst.counter = esw->fdb_table.offloads.drop_root_fc;
2491 dst = &drop_fc_dst;
2492 dst_num++;
2493 }
2494
2495 flow_rule = mlx5_add_flow_rules(table, NULL, &flow_act, dst, dst_num);
2496 err = PTR_ERR_OR_ZERO(flow_rule);
2497 if (err) {
2498 esw_warn(esw->dev,
2499 "fs offloads: Failed to add vport rx drop rule err %d\n",
2500 err);
2501 goto err_flow_rule;
2502 }
2503
2504 esw->fdb_table.offloads.drop_root = table;
2505 esw->fdb_table.offloads.drop_root_rule = flow_rule;
2506 esw_debug(esw->dev, "Created FDB drop root table %#x fc %#x\n",
2507 table->id, dst ? dst->counter->id : 0);
2508 return 0;
2509
2510 err_flow_rule:
2511 /* no need to free drop fc, esw_offloads_steering_cleanup will do it */
2512 mlx5_destroy_flow_table(table);
2513 return err;
2514 }
2515
mlx5_esw_fdb_active(struct mlx5_eswitch * esw)2516 static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw)
2517 {
2518 struct mlx5_vport *vport;
2519 unsigned long i;
2520
2521 mlx5_esw_fdb_drop_destroy(esw);
2522 mlx5_mpfs_enable(esw->dev);
2523
2524 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
2525 if (!vport->adjacent)
2526 continue;
2527 esw_debug(esw->dev, "Connecting vport %d to eswitch\n",
2528 vport->vport);
2529 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true);
2530 }
2531
2532 esw->offloads_inactive = false;
2533 esw_warn(esw->dev, "MPFS/FDB active\n");
2534 }
2535
mlx5_esw_fdb_inactive(struct mlx5_eswitch * esw)2536 static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw)
2537 {
2538 struct mlx5_vport *vport;
2539 unsigned long i;
2540
2541 mlx5_mpfs_disable(esw->dev);
2542 mlx5_esw_fdb_drop_create(esw);
2543
2544 mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
2545 if (!vport->adjacent)
2546 continue;
2547 esw_debug(esw->dev, "Disconnecting vport %u from eswitch\n",
2548 vport->vport);
2549
2550 mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false);
2551 }
2552
2553 esw->offloads_inactive = true;
2554 esw_warn(esw->dev, "MPFS/FDB inactive\n");
2555 }
2556
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2557 static int esw_offloads_start(struct mlx5_eswitch *esw,
2558 struct netlink_ext_ack *extack)
2559 {
2560 int err;
2561
2562 esw_mode_change(esw, MLX5_ESWITCH_OFFLOADS);
2563 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2564 if (err) {
2565 NL_SET_ERR_MSG_MOD(extack,
2566 "Failed setting eswitch to offloads");
2567 esw_mode_change(esw, MLX5_ESWITCH_LEGACY);
2568 return err;
2569 }
2570 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2571 if (mlx5_eswitch_inline_mode_get(esw,
2572 &esw->offloads.inline_mode)) {
2573 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2574 NL_SET_ERR_MSG_MOD(extack,
2575 "Inline mode is different between vports");
2576 }
2577 }
2578 return 0;
2579 }
2580
mlx5_esw_offloads_rep_remove(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2581 void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw,
2582 const struct mlx5_vport *vport)
2583 {
2584 struct mlx5_eswitch_rep *rep = xa_load(&esw->offloads.vport_reps,
2585 vport->vport);
2586
2587 if (!rep)
2588 return;
2589 xa_erase(&esw->offloads.vport_reps, vport->vport);
2590 kfree(rep);
2591 }
2592
mlx5_esw_offloads_rep_add(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2593 int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw,
2594 const struct mlx5_vport *vport)
2595 {
2596 struct mlx5_eswitch_rep *rep;
2597 int rep_type;
2598 int err;
2599
2600 rep = kzalloc_obj(*rep);
2601 if (!rep)
2602 return -ENOMEM;
2603
2604 rep->vport = vport->vport;
2605 rep->vport_index = vport->index;
2606 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2607 if (!esw->offloads.rep_ops[rep_type]) {
2608 atomic_set(&rep->rep_data[rep_type].state,
2609 REP_UNREGISTERED);
2610 continue;
2611 }
2612 /* Dynamic/delegated vports add their representors after
2613 * mlx5_eswitch_register_vport_reps, so mark them as registered
2614 * for them to be loaded later with the others.
2615 */
2616 rep->esw = esw;
2617 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2618 }
2619 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2620 if (err)
2621 goto insert_err;
2622
2623 return 0;
2624
2625 insert_err:
2626 kfree(rep);
2627 return err;
2628 }
2629
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2630 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2631 struct mlx5_eswitch_rep *rep)
2632 {
2633 xa_erase(&esw->offloads.vport_reps, rep->vport);
2634 kfree(rep);
2635 }
2636
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2637 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2638 {
2639 struct mlx5_eswitch_rep *rep;
2640 unsigned long i;
2641
2642 mlx5_esw_for_each_rep(esw, i, rep)
2643 mlx5_esw_offloads_rep_cleanup(esw, rep);
2644 xa_destroy(&esw->offloads.vport_reps);
2645 }
2646
esw_offloads_init_reps(struct mlx5_eswitch * esw)2647 static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2648 {
2649 struct mlx5_vport *vport;
2650 unsigned long i;
2651 int err;
2652
2653 xa_init(&esw->offloads.vport_reps);
2654
2655 mlx5_esw_for_each_vport(esw, i, vport) {
2656 err = mlx5_esw_offloads_rep_add(esw, vport);
2657 if (err)
2658 goto err;
2659 }
2660 return 0;
2661
2662 err:
2663 esw_offloads_cleanup_reps(esw);
2664 return err;
2665 }
2666
esw_port_metadata_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)2667 static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2668 struct devlink_param_gset_ctx *ctx,
2669 struct netlink_ext_ack *extack)
2670 {
2671 struct mlx5_core_dev *dev = devlink_priv(devlink);
2672 struct mlx5_eswitch *esw = dev->priv.eswitch;
2673 int err = 0;
2674
2675 down_write(&esw->mode_lock);
2676 if (mlx5_esw_is_fdb_created(esw)) {
2677 err = -EBUSY;
2678 goto done;
2679 }
2680 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2681 err = -EOPNOTSUPP;
2682 goto done;
2683 }
2684 if (ctx->val.vbool)
2685 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2686 else
2687 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2688 done:
2689 up_write(&esw->mode_lock);
2690 return err;
2691 }
2692
esw_port_metadata_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)2693 static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2694 struct devlink_param_gset_ctx *ctx,
2695 struct netlink_ext_ack *extack)
2696 {
2697 struct mlx5_core_dev *dev = devlink_priv(devlink);
2698
2699 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch);
2700 return 0;
2701 }
2702
esw_port_metadata_validate(struct devlink * devlink,u32 id,union devlink_param_value val,struct netlink_ext_ack * extack)2703 static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2704 union devlink_param_value val,
2705 struct netlink_ext_ack *extack)
2706 {
2707 struct mlx5_core_dev *dev = devlink_priv(devlink);
2708 u8 esw_mode;
2709
2710 esw_mode = mlx5_eswitch_mode(dev);
2711 if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2712 NL_SET_ERR_MSG_MOD(extack,
2713 "E-Switch must either disabled or non switchdev mode");
2714 return -EBUSY;
2715 }
2716 return 0;
2717 }
2718
2719 static const struct devlink_param esw_devlink_params[] = {
2720 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2721 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2722 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2723 esw_port_metadata_get,
2724 esw_port_metadata_set,
2725 esw_port_metadata_validate),
2726 };
2727
esw_offloads_init(struct mlx5_eswitch * esw)2728 int esw_offloads_init(struct mlx5_eswitch *esw)
2729 {
2730 int err;
2731
2732 err = esw_offloads_init_reps(esw);
2733 if (err)
2734 return err;
2735
2736 if (MLX5_ESWITCH_MANAGER(esw->dev) &&
2737 mlx5_esw_vport_match_metadata_supported(esw))
2738 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2739
2740 err = devl_params_register(priv_to_devlink(esw->dev),
2741 esw_devlink_params,
2742 ARRAY_SIZE(esw_devlink_params));
2743 if (err)
2744 goto err_params;
2745
2746 return 0;
2747
2748 err_params:
2749 esw_offloads_cleanup_reps(esw);
2750 return err;
2751 }
2752
esw_offloads_cleanup(struct mlx5_eswitch * esw)2753 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2754 {
2755 devl_params_unregister(priv_to_devlink(esw->dev),
2756 esw_devlink_params,
2757 ARRAY_SIZE(esw_devlink_params));
2758 esw_offloads_cleanup_reps(esw);
2759 }
2760
__esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2761 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
2762 struct mlx5_eswitch_rep *rep, u8 rep_type)
2763 {
2764 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2765 REP_REGISTERED, REP_LOADED) == REP_REGISTERED)
2766 return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2767
2768 return 0;
2769 }
2770
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2771 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2772 struct mlx5_eswitch_rep *rep, u8 rep_type)
2773 {
2774 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2775 REP_LOADED, REP_REGISTERED) == REP_LOADED) {
2776 if (rep_type == REP_ETH)
2777 __esw_offloads_unload_rep(esw, rep, REP_IB);
2778 esw->offloads.rep_ops[rep_type]->unload(rep);
2779 }
2780 }
2781
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2782 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2783 {
2784 struct mlx5_eswitch_rep *rep;
2785 unsigned long i;
2786
2787 mlx5_esw_for_each_rep(esw, i, rep)
2788 __esw_offloads_unload_rep(esw, rep, rep_type);
2789 }
2790
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2791 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2792 {
2793 struct mlx5_eswitch_rep *rep;
2794 int rep_type;
2795 int err;
2796
2797 rep = mlx5_eswitch_get_rep(esw, vport_num);
2798 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2799 err = __esw_offloads_load_rep(esw, rep, rep_type);
2800 if (err)
2801 goto err_reps;
2802 }
2803
2804 return 0;
2805
2806 err_reps:
2807 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2808 for (--rep_type; rep_type >= 0; rep_type--)
2809 __esw_offloads_unload_rep(esw, rep, rep_type);
2810 return err;
2811 }
2812
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2813 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2814 {
2815 struct mlx5_eswitch_rep *rep;
2816 int rep_type;
2817
2818 rep = mlx5_eswitch_get_rep(esw, vport_num);
2819 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2820 __esw_offloads_unload_rep(esw, rep, rep_type);
2821 }
2822
mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2823 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2824 {
2825 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2826 return 0;
2827
2828 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2829 }
2830
mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2831 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2832 {
2833 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2834 return;
2835
2836 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2837 }
2838
mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport,struct mlx5_devlink_port * dl_port,u32 controller,u32 sfnum)2839 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2840 struct mlx5_devlink_port *dl_port,
2841 u32 controller, u32 sfnum)
2842 {
2843 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2844 }
2845
mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2846 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2847 {
2848 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2849 }
2850
mlx5_esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2851 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2852 {
2853 int err;
2854
2855 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2856 return 0;
2857
2858 err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2859 if (err)
2860 return err;
2861
2862 err = mlx5_esw_offloads_rep_load(esw, vport->vport);
2863 if (err)
2864 goto load_err;
2865 return err;
2866
2867 load_err:
2868 mlx5_esw_offloads_devlink_port_unregister(vport);
2869 return err;
2870 }
2871
mlx5_esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2872 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2873 {
2874 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2875 return;
2876
2877 mlx5_esw_offloads_rep_unload(esw, vport->vport);
2878
2879 mlx5_esw_offloads_devlink_port_unregister(vport);
2880 }
2881
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2882 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2883 struct mlx5_core_dev *slave)
2884 {
2885 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2886 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2887 struct mlx5_flow_root_namespace *root;
2888 struct mlx5_flow_namespace *ns;
2889 int err;
2890
2891 MLX5_SET(set_flow_table_root_in, in, opcode,
2892 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2893 MLX5_SET(set_flow_table_root_in, in, table_type,
2894 FS_FT_FDB);
2895
2896 if (master) {
2897 ns = mlx5_get_flow_namespace(master,
2898 MLX5_FLOW_NAMESPACE_FDB);
2899 root = find_root(&ns->node);
2900 mutex_lock(&root->chain_lock);
2901 MLX5_SET(set_flow_table_root_in, in,
2902 table_eswitch_owner_vhca_id_valid, 1);
2903 MLX5_SET(set_flow_table_root_in, in,
2904 table_eswitch_owner_vhca_id,
2905 MLX5_CAP_GEN(master, vhca_id));
2906 MLX5_SET(set_flow_table_root_in, in, table_id,
2907 root->root_ft->id);
2908 } else {
2909 ns = mlx5_get_flow_namespace(slave,
2910 MLX5_FLOW_NAMESPACE_FDB);
2911 root = find_root(&ns->node);
2912 mutex_lock(&root->chain_lock);
2913 MLX5_SET(set_flow_table_root_in, in, table_id,
2914 root->root_ft->id);
2915 }
2916
2917 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2918 mutex_unlock(&root->chain_lock);
2919
2920 return err;
2921 }
2922
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2923 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2924 struct mlx5_core_dev *slave,
2925 struct mlx5_vport *vport,
2926 struct mlx5_flow_table *acl)
2927 {
2928 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2929 struct mlx5_flow_handle *flow_rule = NULL;
2930 struct mlx5_flow_destination dest = {};
2931 struct mlx5_flow_act flow_act = {};
2932 struct mlx5_flow_spec *spec;
2933 int err = 0;
2934 void *misc;
2935
2936 spec = kvzalloc_obj(*spec);
2937 if (!spec)
2938 return -ENOMEM;
2939
2940 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2941 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2942 misc_parameters);
2943 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2944 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2945
2946 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2947 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2948 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2949 source_eswitch_owner_vhca_id);
2950
2951 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2952 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2953 dest.vport.num = slave->priv.eswitch->manager_vport;
2954 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2955 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2956
2957 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2958 &dest, 1);
2959 if (IS_ERR(flow_rule)) {
2960 err = PTR_ERR(flow_rule);
2961 } else {
2962 err = xa_insert(&vport->egress.offloads.bounce_rules,
2963 slave_index, flow_rule, GFP_KERNEL);
2964 if (err)
2965 mlx5_del_flow_rules(flow_rule);
2966 }
2967
2968 kvfree(spec);
2969 return err;
2970 }
2971
esw_master_egress_create_resources(struct mlx5_eswitch * esw,struct mlx5_flow_namespace * egress_ns,struct mlx5_vport * vport,size_t count)2972 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2973 struct mlx5_flow_namespace *egress_ns,
2974 struct mlx5_vport *vport, size_t count)
2975 {
2976 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2977 struct mlx5_flow_table_attr ft_attr = {
2978 .max_fte = count, .prio = 0, .level = 0,
2979 };
2980 struct mlx5_flow_table *acl;
2981 struct mlx5_flow_group *g;
2982 void *match_criteria;
2983 u32 *flow_group_in;
2984 int err;
2985
2986 if (vport->egress.acl)
2987 return 0;
2988
2989 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2990 if (!flow_group_in)
2991 return -ENOMEM;
2992
2993 if (vport->vport || mlx5_core_is_ecpf(esw->dev))
2994 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2995
2996 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2997 if (IS_ERR(acl)) {
2998 err = PTR_ERR(acl);
2999 goto out;
3000 }
3001
3002 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
3003 match_criteria);
3004 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
3005 misc_parameters.source_port);
3006 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
3007 misc_parameters.source_eswitch_owner_vhca_id);
3008 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
3009 MLX5_MATCH_MISC_PARAMETERS);
3010
3011 MLX5_SET(create_flow_group_in, flow_group_in,
3012 source_eswitch_owner_vhca_id_valid, 1);
3013 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
3014 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
3015
3016 g = mlx5_create_flow_group(acl, flow_group_in);
3017 if (IS_ERR(g)) {
3018 err = PTR_ERR(g);
3019 goto err_group;
3020 }
3021
3022 vport->egress.acl = acl;
3023 vport->egress.offloads.bounce_grp = g;
3024 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
3025 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
3026
3027 kvfree(flow_group_in);
3028
3029 return 0;
3030
3031 err_group:
3032 mlx5_destroy_flow_table(acl);
3033 out:
3034 kvfree(flow_group_in);
3035 return err;
3036 }
3037
esw_master_egress_destroy_resources(struct mlx5_vport * vport)3038 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
3039 {
3040 if (!xa_empty(&vport->egress.offloads.bounce_rules))
3041 return;
3042 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp);
3043 vport->egress.offloads.bounce_grp = NULL;
3044 mlx5_destroy_flow_table(vport->egress.acl);
3045 vport->egress.acl = NULL;
3046 }
3047
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,size_t count)3048 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
3049 struct mlx5_core_dev *slave, size_t count)
3050 {
3051 struct mlx5_eswitch *esw = master->priv.eswitch;
3052 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
3053 struct mlx5_flow_namespace *egress_ns;
3054 struct mlx5_vport *vport;
3055 int err;
3056
3057 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3058 if (IS_ERR(vport))
3059 return PTR_ERR(vport);
3060
3061 egress_ns = mlx5_get_flow_vport_namespace(master,
3062 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
3063 vport->index);
3064 if (!egress_ns)
3065 return -EINVAL;
3066
3067 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
3068 return 0;
3069
3070 err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
3071 if (err)
3072 return err;
3073
3074 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index))
3075 return -EINVAL;
3076
3077 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl);
3078 if (err)
3079 goto err_rule;
3080
3081 return 0;
3082
3083 err_rule:
3084 esw_master_egress_destroy_resources(vport);
3085 return err;
3086 }
3087
esw_unset_master_egress_rule(struct mlx5_core_dev * dev,struct mlx5_core_dev * slave_dev)3088 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
3089 struct mlx5_core_dev *slave_dev)
3090 {
3091 struct mlx5_vport *vport;
3092
3093 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
3094 dev->priv.eswitch->manager_vport);
3095
3096 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
3097
3098 if (xa_empty(&vport->egress.offloads.bounce_rules)) {
3099 esw_acl_egress_ofld_cleanup(vport);
3100 xa_destroy(&vport->egress.offloads.bounce_rules);
3101 }
3102 }
3103
mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw,int max_slaves)3104 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
3105 struct mlx5_eswitch *slave_esw, int max_slaves)
3106 {
3107 int err;
3108
3109 err = esw_set_slave_root_fdb(master_esw->dev,
3110 slave_esw->dev);
3111 if (err)
3112 return err;
3113
3114 err = esw_set_master_egress_rule(master_esw->dev,
3115 slave_esw->dev, max_slaves);
3116 if (err)
3117 goto err_acl;
3118
3119 return err;
3120
3121 err_acl:
3122 esw_set_slave_root_fdb(NULL, slave_esw->dev);
3123 return err;
3124 }
3125
mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)3126 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
3127 struct mlx5_eswitch *slave_esw)
3128 {
3129 esw_set_slave_root_fdb(NULL, slave_esw->dev);
3130 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev);
3131 }
3132
3133 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
3134 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
3135
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)3136 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
3137 struct mlx5_eswitch *peer_esw)
3138 {
3139 const struct mlx5_eswitch_rep_ops *ops;
3140 struct mlx5_eswitch_rep *rep;
3141 unsigned long i;
3142 u8 rep_type;
3143
3144 mlx5_esw_for_each_rep(esw, i, rep) {
3145 rep_type = NUM_REP_TYPES;
3146 while (rep_type--) {
3147 ops = esw->offloads.rep_ops[rep_type];
3148 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3149 ops->event)
3150 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
3151 }
3152 }
3153 }
3154
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)3155 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
3156 struct mlx5_eswitch *peer_esw)
3157 {
3158 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3159 mlx5e_tc_clean_fdb_peer_flows(esw);
3160 #endif
3161 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
3162 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev);
3163 }
3164
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)3165 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
3166 struct mlx5_eswitch *peer_esw)
3167 {
3168 const struct mlx5_eswitch_rep_ops *ops;
3169 struct mlx5_eswitch_rep *rep;
3170 unsigned long i;
3171 u8 rep_type;
3172 int err;
3173
3174 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
3175 if (err)
3176 return err;
3177
3178 mlx5_esw_for_each_rep(esw, i, rep) {
3179 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
3180 ops = esw->offloads.rep_ops[rep_type];
3181 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3182 ops->event) {
3183 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
3184 if (err)
3185 goto err_out;
3186 }
3187 }
3188 }
3189
3190 return 0;
3191
3192 err_out:
3193 mlx5_esw_offloads_unpair(esw, peer_esw);
3194 return err;
3195 }
3196
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)3197 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
3198 struct mlx5_eswitch *peer_esw,
3199 bool pair)
3200 {
3201 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
3202 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
3203 struct mlx5_flow_root_namespace *peer_ns;
3204 struct mlx5_flow_root_namespace *ns;
3205 int err;
3206
3207 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
3208 ns = esw->dev->priv.steering->fdb_root_ns;
3209
3210 if (pair) {
3211 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
3212 if (err)
3213 return err;
3214
3215 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id);
3216 if (err) {
3217 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
3218 return err;
3219 }
3220 } else {
3221 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
3222 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id);
3223 }
3224
3225 return 0;
3226 }
3227
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)3228 static int mlx5_esw_offloads_devcom_event(int event,
3229 void *my_data,
3230 void *event_data)
3231 {
3232 struct mlx5_eswitch *esw = my_data;
3233 struct mlx5_eswitch *peer_esw = event_data;
3234 u16 esw_i, peer_esw_i;
3235 bool esw_paired;
3236 int err;
3237
3238 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
3239 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
3240 esw_paired = !!xa_load(&esw->paired, peer_esw_i);
3241
3242 switch (event) {
3243 case ESW_OFFLOADS_DEVCOM_PAIR:
3244 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
3245 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
3246 break;
3247
3248 if (esw_paired)
3249 break;
3250
3251 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
3252 if (err)
3253 goto err_out;
3254
3255 err = mlx5_esw_offloads_pair(esw, peer_esw);
3256 if (err)
3257 goto err_peer;
3258
3259 err = mlx5_esw_offloads_pair(peer_esw, esw);
3260 if (err)
3261 goto err_pair;
3262
3263 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL);
3264 if (err)
3265 goto err_xa;
3266
3267 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL);
3268 if (err)
3269 goto err_peer_xa;
3270
3271 esw->num_peers++;
3272 peer_esw->num_peers++;
3273 mlx5_devcom_comp_set_ready(esw->devcom, true);
3274 break;
3275
3276 case ESW_OFFLOADS_DEVCOM_UNPAIR:
3277 if (!esw_paired)
3278 break;
3279
3280 peer_esw->num_peers--;
3281 esw->num_peers--;
3282 if (!esw->num_peers && !peer_esw->num_peers)
3283 mlx5_devcom_comp_set_ready(esw->devcom, false);
3284 xa_erase(&peer_esw->paired, esw_i);
3285 xa_erase(&esw->paired, peer_esw_i);
3286 mlx5_esw_offloads_unpair(peer_esw, esw);
3287 mlx5_esw_offloads_unpair(esw, peer_esw);
3288 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3289 break;
3290 }
3291
3292 return 0;
3293
3294 err_peer_xa:
3295 xa_erase(&esw->paired, peer_esw_i);
3296 err_xa:
3297 mlx5_esw_offloads_unpair(peer_esw, esw);
3298 err_pair:
3299 mlx5_esw_offloads_unpair(esw, peer_esw);
3300 err_peer:
3301 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3302 err_out:
3303 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3304 event, err);
3305 return err;
3306 }
3307
mlx5_esw_offloads_devcom_init(struct mlx5_eswitch * esw,const struct mlx5_devcom_match_attr * attr)3308 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw,
3309 const struct mlx5_devcom_match_attr *attr)
3310 {
3311 int i;
3312
3313 for (i = 0; i < MLX5_MAX_PORTS; i++)
3314 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]);
3315 mutex_init(&esw->offloads.peer_mutex);
3316
3317 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3318 return;
3319
3320 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) &&
3321 !mlx5_lag_is_supported(esw->dev))
3322 return;
3323
3324 xa_init(&esw->paired);
3325 xa_init(&esw->fdb_table.offloads.peer_miss_rules);
3326 esw->num_peers = 0;
3327 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc,
3328 MLX5_DEVCOM_ESW_OFFLOADS,
3329 attr,
3330 mlx5_esw_offloads_devcom_event,
3331 esw);
3332 if (!esw->devcom)
3333 return;
3334
3335 mlx5_devcom_send_event(esw->devcom,
3336 ESW_OFFLOADS_DEVCOM_PAIR,
3337 ESW_OFFLOADS_DEVCOM_UNPAIR,
3338 esw);
3339 }
3340
mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)3341 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3342 {
3343 if (!esw->devcom)
3344 return;
3345
3346 mlx5_devcom_send_event(esw->devcom,
3347 ESW_OFFLOADS_DEVCOM_UNPAIR,
3348 ESW_OFFLOADS_DEVCOM_UNPAIR,
3349 esw);
3350
3351 mlx5_devcom_unregister_component(esw->devcom);
3352 xa_destroy(&esw->paired);
3353 xa_destroy(&esw->fdb_table.offloads.peer_miss_rules);
3354 esw->devcom = NULL;
3355 }
3356
mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch * esw)3357 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3358 {
3359 return mlx5_devcom_comp_is_ready(esw->devcom);
3360 }
3361
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)3362 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3363 {
3364 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3365 return false;
3366
3367 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3368 MLX5_FDB_TO_VPORT_REG_C_0))
3369 return false;
3370
3371 return true;
3372 }
3373
3374 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
3375
3376 /* Share the same metadata for uplink's. This is fine because:
3377 * (a) In shared FDB mode (LAG) both uplink's are treated the
3378 * same and tagged with the same metadata.
3379 * (b) In non shared FDB mode, packets from physical port0
3380 * cannot hit eswitch of PF1 and vice versa.
3381 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)3382 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3383 {
3384 return MLX5_ESW_METADATA_RSVD_UPLINK;
3385 }
3386
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)3387 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3388 {
3389 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3390 /* Reserve 0xf for internal port offload */
3391 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3392 u32 pf_num;
3393 int id;
3394
3395 /* Only 4 bits of pf_num */
3396 pf_num = mlx5_get_dev_index(esw->dev);
3397 if (pf_num > max_pf_num)
3398 return 0;
3399
3400 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3401 /* Use only non-zero vport_id (2-4095) for all PF's */
3402 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3403 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3404 vport_end_ida, GFP_KERNEL);
3405 if (id < 0)
3406 return 0;
3407 id = (pf_num << ESW_VPORT_BITS) | id;
3408 return id;
3409 }
3410
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)3411 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3412 {
3413 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3414
3415 /* Metadata contains only 12 bits of actual ida id */
3416 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
3417 }
3418
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3419 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3420 struct mlx5_vport *vport)
3421 {
3422 if (vport->vport == MLX5_VPORT_UPLINK)
3423 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3424 else
3425 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3426
3427 vport->metadata = vport->default_metadata;
3428 return vport->metadata ? 0 : -ENOSPC;
3429 }
3430
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3431 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3432 struct mlx5_vport *vport)
3433 {
3434 if (!vport->default_metadata)
3435 return;
3436
3437 if (vport->vport == MLX5_VPORT_UPLINK)
3438 return;
3439
3440 WARN_ON(vport->metadata != vport->default_metadata);
3441 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
3442 }
3443
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)3444 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3445 {
3446 struct mlx5_vport *vport;
3447 unsigned long i;
3448
3449 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3450 return;
3451
3452 mlx5_esw_for_each_vport(esw, i, vport)
3453 esw_offloads_vport_metadata_cleanup(esw, vport);
3454 }
3455
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3456 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3457 {
3458 struct mlx5_vport *vport;
3459 unsigned long i;
3460 int err;
3461
3462 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3463 return 0;
3464
3465 mlx5_esw_for_each_vport(esw, i, vport) {
3466 err = esw_offloads_vport_metadata_setup(esw, vport);
3467 if (err)
3468 goto metadata_err;
3469 }
3470
3471 return 0;
3472
3473 metadata_err:
3474 esw_offloads_metadata_uninit(esw);
3475 return err;
3476 }
3477
3478 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3479 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3480 struct mlx5_vport *vport)
3481 {
3482 int err;
3483
3484 err = esw_acl_ingress_ofld_setup(esw, vport);
3485 if (err)
3486 return err;
3487
3488 err = esw_acl_egress_ofld_setup(esw, vport);
3489 if (err)
3490 goto egress_err;
3491
3492 return 0;
3493
3494 egress_err:
3495 esw_acl_ingress_ofld_cleanup(esw, vport);
3496 return err;
3497 }
3498
3499 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3500 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3501 struct mlx5_vport *vport)
3502 {
3503 esw_acl_egress_ofld_cleanup(vport);
3504 esw_acl_ingress_ofld_cleanup(esw, vport);
3505 }
3506
esw_create_offloads_acl_tables(struct mlx5_eswitch * esw)3507 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3508 {
3509 struct mlx5_vport *uplink, *manager;
3510 int ret;
3511
3512 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3513 if (IS_ERR(uplink))
3514 return PTR_ERR(uplink);
3515
3516 ret = esw_vport_create_offloads_acl_tables(esw, uplink);
3517 if (ret)
3518 return ret;
3519
3520 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3521 if (IS_ERR(manager)) {
3522 ret = PTR_ERR(manager);
3523 goto err_manager;
3524 }
3525
3526 ret = esw_vport_create_offloads_acl_tables(esw, manager);
3527 if (ret)
3528 goto err_manager;
3529
3530 return 0;
3531
3532 err_manager:
3533 esw_vport_destroy_offloads_acl_tables(esw, uplink);
3534 return ret;
3535 }
3536
esw_destroy_offloads_acl_tables(struct mlx5_eswitch * esw)3537 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3538 {
3539 struct mlx5_vport *vport;
3540
3541 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3542 if (!IS_ERR(vport))
3543 esw_vport_destroy_offloads_acl_tables(esw, vport);
3544
3545 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3546 if (!IS_ERR(vport))
3547 esw_vport_destroy_offloads_acl_tables(esw, vport);
3548 }
3549
mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch * esw)3550 int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw)
3551 {
3552 struct mlx5_eswitch_rep *rep;
3553 unsigned long i;
3554 int ret;
3555
3556 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3557 return 0;
3558
3559 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3560 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3561 return 0;
3562
3563 ret = __esw_offloads_load_rep(esw, rep, REP_IB);
3564 if (ret)
3565 return ret;
3566
3567 mlx5_esw_for_each_rep(esw, i, rep) {
3568 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3569 __esw_offloads_load_rep(esw, rep, REP_IB);
3570 }
3571
3572 return 0;
3573 }
3574
esw_offloads_steering_init(struct mlx5_eswitch * esw)3575 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3576 {
3577 struct mlx5_esw_indir_table *indir;
3578 int err;
3579
3580 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3581 mutex_init(&esw->fdb_table.offloads.vports.lock);
3582 hash_init(esw->fdb_table.offloads.vports.table);
3583 atomic64_set(&esw->user_count, 0);
3584
3585 indir = mlx5_esw_indir_table_init();
3586 if (IS_ERR(indir)) {
3587 err = PTR_ERR(indir);
3588 goto create_indir_err;
3589 }
3590 esw->fdb_table.offloads.indir = indir;
3591
3592 err = esw_create_offloads_acl_tables(esw);
3593 if (err)
3594 goto create_acl_err;
3595
3596 err = esw_create_offloads_table(esw);
3597 if (err)
3598 goto create_offloads_err;
3599
3600 err = esw_create_restore_table(esw);
3601 if (err)
3602 goto create_restore_err;
3603
3604 err = esw_create_offloads_fdb_tables(esw);
3605 if (err)
3606 goto create_fdb_err;
3607
3608 err = esw_create_vport_rx_group(esw);
3609 if (err)
3610 goto create_fg_err;
3611
3612 err = esw_create_vport_rx_drop_group(esw);
3613 if (err)
3614 goto create_rx_drop_fg_err;
3615
3616 err = esw_create_vport_rx_drop_rule(esw);
3617 if (err)
3618 goto create_rx_drop_rule_err;
3619
3620 return 0;
3621
3622 create_rx_drop_rule_err:
3623 esw_destroy_vport_rx_drop_group(esw);
3624 create_rx_drop_fg_err:
3625 esw_destroy_vport_rx_group(esw);
3626 create_fg_err:
3627 esw_destroy_offloads_fdb_tables(esw);
3628 create_fdb_err:
3629 esw_destroy_restore_table(esw);
3630 create_restore_err:
3631 esw_destroy_offloads_table(esw);
3632 create_offloads_err:
3633 esw_destroy_offloads_acl_tables(esw);
3634 create_acl_err:
3635 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3636 create_indir_err:
3637 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3638 return err;
3639 }
3640
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3641 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3642 {
3643 mlx5_esw_fdb_drop_destroy(esw);
3644 if (esw->fdb_table.offloads.drop_root_fc)
3645 mlx5_fc_destroy(esw->dev, esw->fdb_table.offloads.drop_root_fc);
3646 esw->fdb_table.offloads.drop_root_fc = NULL;
3647 esw_destroy_vport_rx_drop_rule(esw);
3648 esw_destroy_vport_rx_drop_group(esw);
3649 esw_destroy_vport_rx_group(esw);
3650 esw_destroy_offloads_fdb_tables(esw);
3651 esw_destroy_restore_table(esw);
3652 esw_destroy_offloads_table(esw);
3653 esw_destroy_offloads_acl_tables(esw);
3654 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3655 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3656 }
3657
3658 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,int work_gen,const u32 * out)3659 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen,
3660 const u32 *out)
3661 {
3662 struct devlink *devlink;
3663 bool host_pf_disabled;
3664 u16 new_num_vfs;
3665
3666 devlink = priv_to_devlink(esw->dev);
3667 devl_lock(devlink);
3668
3669 /* Stale work from one or more mode changes ago. Bail out. */
3670 if (work_gen != atomic_read(&esw->esw_funcs.generation))
3671 goto unlock;
3672
3673 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3674 host_params_context.host_num_of_vfs);
3675 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3676 host_params_context.host_pf_disabled);
3677
3678 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3679 goto unlock;
3680
3681 /* Number of VFs can only change from "0 to x" or "x to 0". */
3682 if (esw->esw_funcs.num_vfs > 0) {
3683 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3684 } else {
3685 int err;
3686
3687 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3688 MLX5_VPORT_UC_ADDR_CHANGE);
3689 if (err) {
3690 devl_unlock(devlink);
3691 return;
3692 }
3693 }
3694 esw->esw_funcs.num_vfs = new_num_vfs;
3695 unlock:
3696 devl_unlock(devlink);
3697 }
3698
esw_functions_changed_event_handler(struct work_struct * work)3699 static void esw_functions_changed_event_handler(struct work_struct *work)
3700 {
3701 struct mlx5_host_work *host_work;
3702 struct mlx5_eswitch *esw;
3703 const u32 *out;
3704
3705 host_work = container_of(work, struct mlx5_host_work, work);
3706 esw = host_work->esw;
3707
3708 out = mlx5_esw_query_functions(esw->dev);
3709 if (IS_ERR(out))
3710 goto out;
3711
3712 esw_vfs_changed_event_handler(esw, host_work->work_gen, out);
3713 kvfree(out);
3714 out:
3715 kfree(host_work);
3716 }
3717
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3718 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3719 {
3720 struct mlx5_esw_functions *esw_funcs;
3721 struct mlx5_host_work *host_work;
3722 struct mlx5_eswitch *esw;
3723
3724 host_work = kzalloc_obj(*host_work, GFP_ATOMIC);
3725 if (!host_work)
3726 return NOTIFY_DONE;
3727
3728 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3729 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3730
3731 host_work->esw = esw;
3732 host_work->work_gen = atomic_read(&esw_funcs->generation);
3733
3734 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3735 queue_work(esw->work_queue, &host_work->work);
3736
3737 return NOTIFY_OK;
3738 }
3739
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3740 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3741 {
3742 const u32 *query_host_out;
3743
3744 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3745 return 0;
3746
3747 query_host_out = mlx5_esw_query_functions(esw->dev);
3748 if (IS_ERR(query_host_out))
3749 return PTR_ERR(query_host_out);
3750
3751 /* Mark non local controller with non zero controller number. */
3752 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3753 host_params_context.host_number);
3754 kvfree(query_host_out);
3755 return 0;
3756 }
3757
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3758 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3759 {
3760 /* Local controller is always valid */
3761 if (controller == 0)
3762 return true;
3763
3764 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3765 return false;
3766
3767 /* External host number starts with zero in device */
3768 return (controller == esw->offloads.host_number + 1);
3769 }
3770
esw_offloads_enable(struct mlx5_eswitch * esw)3771 int esw_offloads_enable(struct mlx5_eswitch *esw)
3772 {
3773 u8 mapping_id[MLX5_SW_IMAGE_GUID_MAX_BYTES];
3774 struct mapping_ctx *reg_c0_obj_pool;
3775 struct mlx5_vport *vport;
3776 unsigned long i;
3777 u8 id_len;
3778 int err;
3779
3780 mutex_init(&esw->offloads.termtbl_mutex);
3781 mlx5_esw_adjacent_vhcas_setup(esw);
3782
3783 err = mlx5_rdma_enable_roce(esw->dev);
3784 if (err)
3785 goto err_roce;
3786
3787 err = mlx5_esw_host_number_init(esw);
3788 if (err)
3789 goto err_metadata;
3790
3791 err = esw_offloads_metadata_init(esw);
3792 if (err)
3793 goto err_metadata;
3794
3795 err = esw_set_passing_vport_metadata(esw, true);
3796 if (err)
3797 goto err_vport_metadata;
3798
3799 mlx5_query_nic_sw_system_image_guid(esw->dev, mapping_id, &id_len);
3800
3801 reg_c0_obj_pool = mapping_create_for_id(mapping_id, id_len,
3802 MAPPING_TYPE_CHAIN,
3803 sizeof(struct mlx5_mapped_obj),
3804 ESW_REG_C0_USER_DATA_METADATA_MASK,
3805 true);
3806
3807 if (IS_ERR(reg_c0_obj_pool)) {
3808 err = PTR_ERR(reg_c0_obj_pool);
3809 goto err_pool;
3810 }
3811 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3812
3813 err = esw_offloads_steering_init(esw);
3814 if (err)
3815 goto err_steering_init;
3816
3817 if (esw->offloads_inactive)
3818 mlx5_esw_fdb_inactive(esw);
3819 else
3820 mlx5_esw_fdb_active(esw);
3821
3822 /* Representor will control the vport link state */
3823 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3824 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3825 if (mlx5_core_ec_sriov_enabled(esw->dev))
3826 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3827 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3828
3829 /* Uplink vport rep must load first. */
3830 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3831 if (err)
3832 goto err_uplink;
3833
3834 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3835 if (err)
3836 goto err_vports;
3837
3838 return 0;
3839
3840 err_vports:
3841 /* rollback to legacy, indicates don't unregister the uplink netdev */
3842 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY;
3843 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3844 err_uplink:
3845 esw_offloads_steering_cleanup(esw);
3846 err_steering_init:
3847 mapping_destroy(reg_c0_obj_pool);
3848 err_pool:
3849 esw_set_passing_vport_metadata(esw, false);
3850 err_vport_metadata:
3851 esw_offloads_metadata_uninit(esw);
3852 err_metadata:
3853 mlx5_rdma_disable_roce(esw->dev);
3854 err_roce:
3855 mlx5_esw_adjacent_vhcas_cleanup(esw);
3856 mutex_destroy(&esw->offloads.termtbl_mutex);
3857 return err;
3858 }
3859
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3860 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3861 struct netlink_ext_ack *extack)
3862 {
3863 int err;
3864
3865 esw_mode_change(esw, MLX5_ESWITCH_LEGACY);
3866
3867 /* If changing from switchdev to legacy mode without sriov enabled,
3868 * no need to create legacy fdb.
3869 */
3870 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev))
3871 return 0;
3872
3873 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3874 if (err)
3875 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3876
3877 return err;
3878 }
3879
esw_offloads_disable(struct mlx5_eswitch * esw)3880 void esw_offloads_disable(struct mlx5_eswitch *esw)
3881 {
3882 mlx5_eswitch_disable_pf_vf_vports(esw);
3883 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3884 esw_set_passing_vport_metadata(esw, false);
3885 esw_offloads_steering_cleanup(esw);
3886 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3887 esw_offloads_metadata_uninit(esw);
3888 mlx5_rdma_disable_roce(esw->dev);
3889 mlx5_esw_adjacent_vhcas_cleanup(esw);
3890 /* must be done after vhcas cleanup to avoid adjacent vports connect */
3891 if (esw->offloads_inactive)
3892 mlx5_esw_fdb_active(esw); /* legacy mode always active */
3893 mutex_destroy(&esw->offloads.termtbl_mutex);
3894 }
3895
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3896 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3897 {
3898 switch (mode) {
3899 case DEVLINK_ESWITCH_MODE_LEGACY:
3900 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3901 break;
3902 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3903 case DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE:
3904 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3905 break;
3906 default:
3907 return -EINVAL;
3908 }
3909
3910 return 0;
3911 }
3912
esw_mode_to_devlink(struct mlx5_eswitch * esw,u16 * mode)3913 static int esw_mode_to_devlink(struct mlx5_eswitch *esw, u16 *mode)
3914 {
3915 switch (esw->mode) {
3916 case MLX5_ESWITCH_LEGACY:
3917 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3918 break;
3919 case MLX5_ESWITCH_OFFLOADS:
3920 if (esw->offloads_inactive)
3921 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE;
3922 else
3923 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3924 break;
3925 default:
3926 return -EINVAL;
3927 }
3928
3929 return 0;
3930 }
3931
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3932 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3933 {
3934 switch (mode) {
3935 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3936 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3937 break;
3938 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3939 *mlx5_mode = MLX5_INLINE_MODE_L2;
3940 break;
3941 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3942 *mlx5_mode = MLX5_INLINE_MODE_IP;
3943 break;
3944 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3945 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3946 break;
3947 default:
3948 return -EINVAL;
3949 }
3950
3951 return 0;
3952 }
3953
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3954 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3955 {
3956 switch (mlx5_mode) {
3957 case MLX5_INLINE_MODE_NONE:
3958 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3959 break;
3960 case MLX5_INLINE_MODE_L2:
3961 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3962 break;
3963 case MLX5_INLINE_MODE_IP:
3964 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3965 break;
3966 case MLX5_INLINE_MODE_TCP_UDP:
3967 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3968 break;
3969 default:
3970 return -EINVAL;
3971 }
3972
3973 return 0;
3974 }
3975
mlx5_eswitch_block_mode(struct mlx5_core_dev * dev)3976 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3977 {
3978 struct mlx5_eswitch *esw = dev->priv.eswitch;
3979 int err;
3980
3981 if (!mlx5_esw_allowed(esw))
3982 return 0;
3983
3984 /* Take TC into account */
3985 err = mlx5_esw_try_lock(esw);
3986 if (err < 0)
3987 return err;
3988
3989 esw->offloads.num_block_mode++;
3990 mlx5_esw_unlock(esw);
3991 return 0;
3992 }
3993
mlx5_eswitch_unblock_mode(struct mlx5_core_dev * dev)3994 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3995 {
3996 struct mlx5_eswitch *esw = dev->priv.eswitch;
3997
3998 if (!mlx5_esw_allowed(esw))
3999 return;
4000
4001 down_write(&esw->mode_lock);
4002 esw->offloads.num_block_mode--;
4003 up_write(&esw->mode_lock);
4004 }
4005
4006 /* Returns false only when uplink netdev exists and its netns is different from
4007 * devlink's netns. True for all others so entering switchdev mode is allowed.
4008 */
mlx5_devlink_netdev_netns_immutable_set(struct devlink * devlink,bool immutable)4009 static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink,
4010 bool immutable)
4011 {
4012 struct mlx5_core_dev *mdev = devlink_priv(devlink);
4013 struct net_device *netdev;
4014 bool ret;
4015
4016 netdev = mlx5_uplink_netdev_get(mdev);
4017 if (!netdev)
4018 return true;
4019
4020 rtnl_lock();
4021 netdev->netns_immutable = immutable;
4022 ret = net_eq(dev_net(netdev), devlink_net(devlink));
4023 rtnl_unlock();
4024
4025 mlx5_uplink_netdev_put(mdev, netdev);
4026 return ret;
4027 }
4028
4029 /* Returns true when only changing between active and inactive switchdev mode */
mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch * esw,u16 devlink_mode)4030 static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw,
4031 u16 devlink_mode)
4032 {
4033 /* current mode is not switchdev */
4034 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
4035 return false;
4036
4037 /* new mode is not switchdev */
4038 if (devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV &&
4039 devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE)
4040 return false;
4041
4042 /* already inactive: no change in current state */
4043 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE &&
4044 esw->offloads_inactive)
4045 return false;
4046
4047 /* already active: no change in current state */
4048 if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV &&
4049 !esw->offloads_inactive)
4050 return false;
4051
4052 down_write(&esw->mode_lock);
4053 esw->offloads_inactive = !esw->offloads_inactive;
4054 esw->eswitch_operation_in_progress = true;
4055 up_write(&esw->mode_lock);
4056
4057 if (esw->offloads_inactive)
4058 mlx5_esw_fdb_inactive(esw);
4059 else
4060 mlx5_esw_fdb_active(esw);
4061
4062 down_write(&esw->mode_lock);
4063 esw->eswitch_operation_in_progress = false;
4064 up_write(&esw->mode_lock);
4065 return true;
4066 }
4067
4068 #define MLX5_ESW_HOLD_TIMEOUT_MS 7000
4069 #define MLX5_ESW_HOLD_RETRY_DELAY_MS 500
4070
mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev * dev)4071 void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev)
4072 {
4073 unsigned long timeout;
4074 bool hold_esw = true;
4075
4076 /* Wait for any concurrent eswitch mode transition to complete. */
4077 if (!mlx5_esw_hold(dev)) {
4078 timeout = jiffies + msecs_to_jiffies(MLX5_ESW_HOLD_TIMEOUT_MS);
4079 while (!mlx5_esw_hold(dev)) {
4080 if (!time_before(jiffies, timeout)) {
4081 hold_esw = false;
4082 break;
4083 }
4084 msleep(MLX5_ESW_HOLD_RETRY_DELAY_MS);
4085 }
4086 }
4087 if (hold_esw) {
4088 if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS)
4089 mlx5_core_reps_aux_devs_remove(dev);
4090 mlx5_esw_release(dev);
4091 }
4092 }
4093
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)4094 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
4095 struct netlink_ext_ack *extack)
4096 {
4097 u16 cur_mlx5_mode, mlx5_mode = 0;
4098 struct mlx5_eswitch *esw;
4099 int err = 0;
4100
4101 esw = mlx5_devlink_eswitch_get(devlink);
4102 if (IS_ERR(esw))
4103 return PTR_ERR(esw);
4104
4105 if (mlx5_fw_reset_in_progress(esw->dev)) {
4106 NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset");
4107 return -EBUSY;
4108 }
4109
4110 if (esw_mode_from_devlink(mode, &mlx5_mode))
4111 return -EINVAL;
4112
4113 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && mlx5_get_sd(esw->dev)) {
4114 NL_SET_ERR_MSG_MOD(extack,
4115 "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured.");
4116 return -EPERM;
4117 }
4118
4119 /* Avoid try_lock, active/inactive mode change is not restricted */
4120 if (mlx5_devlink_switchdev_active_mode_change(esw, mode))
4121 return 0;
4122
4123 mlx5_lag_disable_change(esw->dev);
4124 err = mlx5_esw_try_lock(esw);
4125 if (err < 0) {
4126 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
4127 goto enable_lag;
4128 }
4129 cur_mlx5_mode = err;
4130 err = 0;
4131
4132 if (cur_mlx5_mode == mlx5_mode)
4133 goto unlock;
4134
4135 if (esw->offloads.num_block_mode) {
4136 NL_SET_ERR_MSG_MOD(extack,
4137 "Can't change eswitch mode when IPsec SA and/or policies are configured");
4138 err = -EOPNOTSUPP;
4139 goto unlock;
4140 }
4141
4142 esw->eswitch_operation_in_progress = true;
4143 up_write(&esw->mode_lock);
4144
4145 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS &&
4146 !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) {
4147 NL_SET_ERR_MSG_MOD(extack,
4148 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's.");
4149 err = -EINVAL;
4150 goto skip;
4151 }
4152
4153 if (mlx5_mode == MLX5_ESWITCH_LEGACY)
4154 esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY;
4155 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS)
4156 esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY;
4157 mlx5_eswitch_disable_locked(esw);
4158 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) {
4159 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
4160 NL_SET_ERR_MSG_MOD(extack,
4161 "Can't change mode while devlink traps are active");
4162 err = -EOPNOTSUPP;
4163 goto skip;
4164 }
4165 esw->offloads_inactive =
4166 (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE);
4167 err = esw_offloads_start(esw, extack);
4168 } else if (mlx5_mode == MLX5_ESWITCH_LEGACY) {
4169 err = esw_offloads_stop(esw, extack);
4170 } else {
4171 err = -EINVAL;
4172 }
4173
4174 skip:
4175 if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err)
4176 mlx5_devlink_netdev_netns_immutable_set(devlink, false);
4177 down_write(&esw->mode_lock);
4178 esw->eswitch_operation_in_progress = false;
4179 unlock:
4180 mlx5_esw_unlock(esw);
4181 enable_lag:
4182 mlx5_lag_enable_change(esw->dev);
4183 return err;
4184 }
4185
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)4186 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
4187 {
4188 struct mlx5_eswitch *esw;
4189
4190 esw = mlx5_devlink_eswitch_get(devlink);
4191 if (IS_ERR(esw))
4192 return PTR_ERR(esw);
4193
4194 return esw_mode_to_devlink(esw, mode);
4195 }
4196
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)4197 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
4198 struct netlink_ext_ack *extack)
4199 {
4200 struct mlx5_core_dev *dev = esw->dev;
4201 struct mlx5_vport *vport;
4202 u16 err_vport_num = 0;
4203 unsigned long i;
4204 int err = 0;
4205
4206 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
4207 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
4208 if (err) {
4209 err_vport_num = vport->vport;
4210 NL_SET_ERR_MSG_MOD(extack,
4211 "Failed to set min inline on vport");
4212 goto revert_inline_mode;
4213 }
4214 }
4215 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
4216 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
4217 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
4218 if (err) {
4219 err_vport_num = vport->vport;
4220 NL_SET_ERR_MSG_MOD(extack,
4221 "Failed to set min inline on vport");
4222 goto revert_ec_vf_inline_mode;
4223 }
4224 }
4225 }
4226 return 0;
4227
4228 revert_ec_vf_inline_mode:
4229 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
4230 if (vport->vport == err_vport_num)
4231 break;
4232 mlx5_modify_nic_vport_min_inline(dev,
4233 vport->vport,
4234 esw->offloads.inline_mode);
4235 }
4236 revert_inline_mode:
4237 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
4238 if (vport->vport == err_vport_num)
4239 break;
4240 mlx5_modify_nic_vport_min_inline(dev,
4241 vport->vport,
4242 esw->offloads.inline_mode);
4243 }
4244 return err;
4245 }
4246
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)4247 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
4248 struct netlink_ext_ack *extack)
4249 {
4250 struct mlx5_core_dev *dev = devlink_priv(devlink);
4251 struct mlx5_eswitch *esw;
4252 u8 mlx5_mode;
4253 int err;
4254
4255 esw = mlx5_devlink_eswitch_get(devlink);
4256 if (IS_ERR(esw))
4257 return PTR_ERR(esw);
4258
4259 down_write(&esw->mode_lock);
4260
4261 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
4262 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
4263 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
4264 err = 0;
4265 goto out;
4266 }
4267
4268 fallthrough;
4269 case MLX5_CAP_INLINE_MODE_L2:
4270 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
4271 err = -EOPNOTSUPP;
4272 goto out;
4273 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
4274 break;
4275 }
4276
4277 if (atomic64_read(&esw->offloads.num_flows) > 0) {
4278 NL_SET_ERR_MSG_MOD(extack,
4279 "Can't set inline mode when flows are configured");
4280 err = -EOPNOTSUPP;
4281 goto out;
4282 }
4283
4284 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
4285 if (err)
4286 goto out;
4287
4288 esw->eswitch_operation_in_progress = true;
4289 up_write(&esw->mode_lock);
4290
4291 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
4292 if (!err)
4293 esw->offloads.inline_mode = mlx5_mode;
4294
4295 down_write(&esw->mode_lock);
4296 esw->eswitch_operation_in_progress = false;
4297 up_write(&esw->mode_lock);
4298 return 0;
4299
4300 out:
4301 up_write(&esw->mode_lock);
4302 return err;
4303 }
4304
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)4305 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
4306 {
4307 struct mlx5_eswitch *esw;
4308
4309 esw = mlx5_devlink_eswitch_get(devlink);
4310 if (IS_ERR(esw))
4311 return PTR_ERR(esw);
4312
4313 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
4314 }
4315
mlx5_eswitch_block_encap(struct mlx5_core_dev * dev,bool from_fdb)4316 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev, bool from_fdb)
4317 {
4318 struct mlx5_eswitch *esw = dev->priv.eswitch;
4319 enum devlink_eswitch_encap_mode encap;
4320 bool allow_tunnel = false;
4321
4322 if (!mlx5_esw_allowed(esw))
4323 return true;
4324
4325 down_write(&esw->mode_lock);
4326 encap = esw->offloads.encap;
4327 if (esw->mode == MLX5_ESWITCH_LEGACY ||
4328 (encap == DEVLINK_ESWITCH_ENCAP_MODE_NONE && !from_fdb)) {
4329 allow_tunnel = true;
4330 esw->offloads.num_block_encap++;
4331 }
4332 up_write(&esw->mode_lock);
4333
4334 return allow_tunnel;
4335 }
4336
mlx5_eswitch_unblock_encap(struct mlx5_core_dev * dev)4337 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
4338 {
4339 struct mlx5_eswitch *esw = dev->priv.eswitch;
4340
4341 if (!mlx5_esw_allowed(esw))
4342 return;
4343
4344 down_write(&esw->mode_lock);
4345 esw->offloads.num_block_encap--;
4346 up_write(&esw->mode_lock);
4347 }
4348
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)4349 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
4350 enum devlink_eswitch_encap_mode encap,
4351 struct netlink_ext_ack *extack)
4352 {
4353 struct mlx5_core_dev *dev = devlink_priv(devlink);
4354 struct mlx5_eswitch *esw;
4355 int err = 0;
4356
4357 esw = mlx5_devlink_eswitch_get(devlink);
4358 if (IS_ERR(esw))
4359 return PTR_ERR(esw);
4360
4361 down_write(&esw->mode_lock);
4362
4363 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
4364 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
4365 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
4366 err = -EOPNOTSUPP;
4367 goto unlock;
4368 }
4369
4370 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
4371 err = -EOPNOTSUPP;
4372 goto unlock;
4373 }
4374
4375 if (esw->mode == MLX5_ESWITCH_LEGACY) {
4376 esw->offloads.encap = encap;
4377 goto unlock;
4378 }
4379
4380 if (esw->offloads.encap == encap)
4381 goto unlock;
4382
4383 if (atomic64_read(&esw->offloads.num_flows) > 0) {
4384 NL_SET_ERR_MSG_MOD(extack,
4385 "Can't set encapsulation when flows are configured");
4386 err = -EOPNOTSUPP;
4387 goto unlock;
4388 }
4389
4390 if (esw->offloads.num_block_encap) {
4391 NL_SET_ERR_MSG_MOD(extack,
4392 "Can't set encapsulation when IPsec SA and/or policies are configured");
4393 err = -EOPNOTSUPP;
4394 goto unlock;
4395 }
4396
4397 esw->eswitch_operation_in_progress = true;
4398 up_write(&esw->mode_lock);
4399
4400 esw_destroy_offloads_fdb_tables(esw);
4401
4402 esw->offloads.encap = encap;
4403
4404 err = esw_create_offloads_fdb_tables(esw);
4405
4406 if (err) {
4407 NL_SET_ERR_MSG_MOD(extack,
4408 "Failed re-creating fast FDB table");
4409 esw->offloads.encap = !encap;
4410 (void)esw_create_offloads_fdb_tables(esw);
4411 }
4412
4413 down_write(&esw->mode_lock);
4414 esw->eswitch_operation_in_progress = false;
4415
4416 unlock:
4417 up_write(&esw->mode_lock);
4418 return err;
4419 }
4420
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)4421 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4422 enum devlink_eswitch_encap_mode *encap)
4423 {
4424 struct mlx5_eswitch *esw;
4425
4426 esw = mlx5_devlink_eswitch_get(devlink);
4427 if (IS_ERR(esw))
4428 return PTR_ERR(esw);
4429
4430 *encap = esw->offloads.encap;
4431 return 0;
4432 }
4433
4434 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)4435 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4436 {
4437 /* Currently, only ECPF based device has representor for host PF. */
4438 if (vport_num == MLX5_VPORT_PF &&
4439 (!mlx5_core_is_ecpf_esw_manager(esw->dev) ||
4440 !mlx5_esw_host_functions_enabled(esw->dev)))
4441 return false;
4442
4443 if (vport_num == MLX5_VPORT_ECPF &&
4444 !mlx5_ecpf_vport_exists(esw->dev))
4445 return false;
4446
4447 return true;
4448 }
4449
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)4450 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4451 const struct mlx5_eswitch_rep_ops *ops,
4452 u8 rep_type)
4453 {
4454 struct mlx5_eswitch_rep_data *rep_data;
4455 struct mlx5_eswitch_rep *rep;
4456 unsigned long i;
4457
4458 esw->offloads.rep_ops[rep_type] = ops;
4459 mlx5_esw_for_each_rep(esw, i, rep) {
4460 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4461 rep->esw = esw;
4462 rep_data = &rep->rep_data[rep_type];
4463 atomic_set(&rep_data->state, REP_REGISTERED);
4464 }
4465 }
4466 }
4467 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4468
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)4469 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4470 {
4471 struct mlx5_eswitch_rep *rep;
4472 unsigned long i;
4473
4474 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4475 __unload_reps_all_vport(esw, rep_type);
4476
4477 mlx5_esw_for_each_rep(esw, i, rep)
4478 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
4479 }
4480 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4481
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)4482 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4483 {
4484 struct mlx5_eswitch_rep *rep;
4485
4486 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
4487 return rep->rep_data[rep_type].priv;
4488 }
4489
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)4490 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4491 u16 vport,
4492 u8 rep_type)
4493 {
4494 struct mlx5_eswitch_rep *rep;
4495
4496 rep = mlx5_eswitch_get_rep(esw, vport);
4497
4498 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
4499 esw->offloads.rep_ops[rep_type]->get_proto_dev)
4500 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4501 return NULL;
4502 }
4503 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4504
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)4505 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4506 {
4507 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4508 }
4509 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4510
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)4511 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4512 u16 vport)
4513 {
4514 return mlx5_eswitch_get_rep(esw, vport);
4515 }
4516 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4517
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)4518 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4519 {
4520 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4521 }
4522 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4523
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)4524 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4525 {
4526 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4527 }
4528 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4529
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)4530 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4531 u16 vport_num)
4532 {
4533 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4534
4535 if (WARN_ON_ONCE(IS_ERR(vport)))
4536 return 0;
4537
4538 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4539 }
4540 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4541
mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch * esw,struct mlx5_vport * vport)4542 int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw,
4543 struct mlx5_vport *vport)
4544 {
4545 u16 *old_entry, *vhca_map_entry, vhca_id;
4546
4547 if (WARN_ONCE(MLX5_VPORT_INVAL_VHCA_ID(vport),
4548 "vport %d vhca_id is not set", vport->vport)) {
4549 int err;
4550
4551 err = mlx5_vport_get_vhca_id(vport->dev, vport->vport,
4552 &vhca_id);
4553 if (err)
4554 return err;
4555 vport->vhca_id = vhca_id;
4556 }
4557
4558 vhca_id = vport->vhca_id;
4559 vhca_map_entry = kmalloc_obj(*vhca_map_entry);
4560 if (!vhca_map_entry)
4561 return -ENOMEM;
4562
4563 *vhca_map_entry = vport->vport;
4564 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
4565 if (xa_is_err(old_entry)) {
4566 kfree(vhca_map_entry);
4567 return xa_err(old_entry);
4568 }
4569 kfree(old_entry);
4570 return 0;
4571 }
4572
mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch * esw,struct mlx5_vport * vport)4573 void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw,
4574 struct mlx5_vport *vport)
4575 {
4576 u16 *vhca_map_entry;
4577
4578 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vport->vhca_id);
4579 kfree(vhca_map_entry);
4580 }
4581
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)4582 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4583 {
4584 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
4585
4586 if (!res)
4587 return -ENOENT;
4588
4589 *vport_num = *res;
4590 return 0;
4591 }
4592
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)4593 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4594 u16 vport_num)
4595 {
4596 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4597
4598 if (WARN_ON_ONCE(IS_ERR(vport)))
4599 return 0;
4600
4601 return vport->metadata;
4602 }
4603 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4604
mlx5_devlink_port_fn_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)4605 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4606 u8 *hw_addr, int *hw_addr_len,
4607 struct netlink_ext_ack *extack)
4608 {
4609 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4610 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4611
4612 mutex_lock(&esw->state_lock);
4613
4614 mlx5_query_nic_vport_mac_address(esw->dev, vport->vport, true,
4615 vport->info.mac);
4616 ether_addr_copy(hw_addr, vport->info.mac);
4617 *hw_addr_len = ETH_ALEN;
4618 mutex_unlock(&esw->state_lock);
4619 return 0;
4620 }
4621
mlx5_devlink_port_fn_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4622 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4623 const u8 *hw_addr, int hw_addr_len,
4624 struct netlink_ext_ack *extack)
4625 {
4626 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4627 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4628
4629 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr);
4630 }
4631
mlx5_devlink_port_fn_migratable_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4632 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4633 struct netlink_ext_ack *extack)
4634 {
4635 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4636 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4637
4638 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4639 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4640 return -EOPNOTSUPP;
4641 }
4642
4643 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4644 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4645 return -EOPNOTSUPP;
4646 }
4647
4648 mutex_lock(&esw->state_lock);
4649 *is_enabled = vport->info.mig_enabled;
4650 mutex_unlock(&esw->state_lock);
4651 return 0;
4652 }
4653
mlx5_devlink_port_fn_migratable_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4654 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4655 struct netlink_ext_ack *extack)
4656 {
4657 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4658 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4659 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4660 void *query_ctx;
4661 void *hca_caps;
4662 int err;
4663
4664 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4665 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4666 return -EOPNOTSUPP;
4667 }
4668
4669 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4670 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4671 return -EOPNOTSUPP;
4672 }
4673
4674 mutex_lock(&esw->state_lock);
4675
4676 if (vport->info.mig_enabled == enable) {
4677 err = 0;
4678 goto out;
4679 }
4680
4681 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4682 if (!query_ctx) {
4683 err = -ENOMEM;
4684 goto out;
4685 }
4686
4687 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
4688 MLX5_CAP_GENERAL_2);
4689 if (err) {
4690 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4691 goto out_free;
4692 }
4693
4694 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4695 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4696
4697 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport,
4698 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4699 if (err) {
4700 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4701 goto out_free;
4702 }
4703
4704 vport->info.mig_enabled = enable;
4705
4706 out_free:
4707 kfree(query_ctx);
4708 out:
4709 mutex_unlock(&esw->state_lock);
4710 return err;
4711 }
4712
mlx5_devlink_port_fn_roce_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4713 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4714 struct netlink_ext_ack *extack)
4715 {
4716 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4717 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4718
4719 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4720 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4721 return -EOPNOTSUPP;
4722 }
4723
4724 mutex_lock(&esw->state_lock);
4725 *is_enabled = vport->info.roce_enabled;
4726 mutex_unlock(&esw->state_lock);
4727 return 0;
4728 }
4729
mlx5_devlink_port_fn_roce_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4730 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4731 struct netlink_ext_ack *extack)
4732 {
4733 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4734 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4735 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4736 u16 vport_num = vport->vport;
4737 void *query_ctx;
4738 void *hca_caps;
4739 int err;
4740
4741 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4742 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4743 return -EOPNOTSUPP;
4744 }
4745
4746 mutex_lock(&esw->state_lock);
4747
4748 if (vport->info.roce_enabled == enable) {
4749 err = 0;
4750 goto out;
4751 }
4752
4753 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4754 if (!query_ctx) {
4755 err = -ENOMEM;
4756 goto out;
4757 }
4758
4759 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4760 MLX5_CAP_GENERAL);
4761 if (err) {
4762 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4763 goto out_free;
4764 }
4765
4766 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4767 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4768
4769 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
4770 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4771 if (err) {
4772 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4773 goto out_free;
4774 }
4775
4776 vport->info.roce_enabled = enable;
4777
4778 out_free:
4779 kfree(query_ctx);
4780 out:
4781 mutex_unlock(&esw->state_lock);
4782 return err;
4783 }
4784
mlx5_devlink_pf_port_fn_state_get(struct devlink_port * port,enum devlink_port_fn_state * state,enum devlink_port_fn_opstate * opstate,struct netlink_ext_ack * extack)4785 int mlx5_devlink_pf_port_fn_state_get(struct devlink_port *port,
4786 enum devlink_port_fn_state *state,
4787 enum devlink_port_fn_opstate *opstate,
4788 struct netlink_ext_ack *extack)
4789 {
4790 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4791 const u32 *query_out;
4792 bool pf_disabled;
4793
4794 if (vport->vport != MLX5_VPORT_PF) {
4795 NL_SET_ERR_MSG_MOD(extack, "State get is not supported for VF");
4796 return -EOPNOTSUPP;
4797 }
4798
4799 *state = vport->pf_activated ?
4800 DEVLINK_PORT_FN_STATE_ACTIVE : DEVLINK_PORT_FN_STATE_INACTIVE;
4801
4802 query_out = mlx5_esw_query_functions(vport->dev);
4803 if (IS_ERR(query_out))
4804 return PTR_ERR(query_out);
4805
4806 pf_disabled = MLX5_GET(query_esw_functions_out, query_out,
4807 host_params_context.host_pf_disabled);
4808
4809 *opstate = pf_disabled ? DEVLINK_PORT_FN_OPSTATE_DETACHED :
4810 DEVLINK_PORT_FN_OPSTATE_ATTACHED;
4811
4812 kvfree(query_out);
4813 return 0;
4814 }
4815
mlx5_devlink_pf_port_fn_state_set(struct devlink_port * port,enum devlink_port_fn_state state,struct netlink_ext_ack * extack)4816 int mlx5_devlink_pf_port_fn_state_set(struct devlink_port *port,
4817 enum devlink_port_fn_state state,
4818 struct netlink_ext_ack *extack)
4819 {
4820 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4821 struct mlx5_core_dev *dev;
4822
4823 if (vport->vport != MLX5_VPORT_PF) {
4824 NL_SET_ERR_MSG_MOD(extack, "State set is not supported for VF");
4825 return -EOPNOTSUPP;
4826 }
4827
4828 dev = vport->dev;
4829
4830 switch (state) {
4831 case DEVLINK_PORT_FN_STATE_ACTIVE:
4832 return mlx5_esw_host_pf_enable_hca(dev);
4833 case DEVLINK_PORT_FN_STATE_INACTIVE:
4834 return mlx5_esw_host_pf_disable_hca(dev);
4835 default:
4836 return -EOPNOTSUPP;
4837 }
4838 }
4839
4840 int
mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)4841 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4842 struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4843 {
4844 struct mlx5_flow_destination new_dest = {};
4845 struct mlx5_flow_destination old_dest = {};
4846
4847 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4848 return 0;
4849
4850 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4851 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4852
4853 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest);
4854 }
4855
4856 #ifdef CONFIG_XFRM_OFFLOAD
mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4857 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4858 struct netlink_ext_ack *extack)
4859 {
4860 struct mlx5_eswitch *esw;
4861 struct mlx5_vport *vport;
4862 int err = 0;
4863
4864 esw = mlx5_devlink_eswitch_get(port->devlink);
4865 if (IS_ERR(esw))
4866 return PTR_ERR(esw);
4867
4868 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4869 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4870 return -EOPNOTSUPP;
4871 }
4872
4873 vport = mlx5_devlink_port_vport_get(port);
4874
4875 mutex_lock(&esw->state_lock);
4876 if (!vport->enabled) {
4877 err = -EOPNOTSUPP;
4878 goto unlock;
4879 }
4880
4881 *is_enabled = vport->info.ipsec_crypto_enabled;
4882 unlock:
4883 mutex_unlock(&esw->state_lock);
4884 return err;
4885 }
4886
mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4887 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4888 struct netlink_ext_ack *extack)
4889 {
4890 struct mlx5_eswitch *esw;
4891 struct mlx5_vport *vport;
4892 u16 vport_num;
4893 int err;
4894
4895 esw = mlx5_devlink_eswitch_get(port->devlink);
4896 if (IS_ERR(esw))
4897 return PTR_ERR(esw);
4898
4899 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4900 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num);
4901 if (err) {
4902 NL_SET_ERR_MSG_MOD(extack,
4903 "Device doesn't support IPsec crypto");
4904 return err;
4905 }
4906
4907 vport = mlx5_devlink_port_vport_get(port);
4908
4909 mutex_lock(&esw->state_lock);
4910 if (!vport->enabled) {
4911 err = -EOPNOTSUPP;
4912 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4913 goto unlock;
4914 }
4915
4916 if (vport->info.ipsec_crypto_enabled == enable)
4917 goto unlock;
4918
4919 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4920 err = -EBUSY;
4921 goto unlock;
4922 }
4923
4924 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4925 if (err) {
4926 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4927 goto unlock;
4928 }
4929
4930 vport->info.ipsec_crypto_enabled = enable;
4931 if (enable)
4932 esw->enabled_ipsec_vf_count++;
4933 else
4934 esw->enabled_ipsec_vf_count--;
4935 unlock:
4936 mutex_unlock(&esw->state_lock);
4937 return err;
4938 }
4939
mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4940 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4941 struct netlink_ext_ack *extack)
4942 {
4943 struct mlx5_eswitch *esw;
4944 struct mlx5_vport *vport;
4945 int err = 0;
4946
4947 esw = mlx5_devlink_eswitch_get(port->devlink);
4948 if (IS_ERR(esw))
4949 return PTR_ERR(esw);
4950
4951 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4952 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4953 return -EOPNOTSUPP;
4954 }
4955
4956 vport = mlx5_devlink_port_vport_get(port);
4957
4958 mutex_lock(&esw->state_lock);
4959 if (!vport->enabled) {
4960 err = -EOPNOTSUPP;
4961 goto unlock;
4962 }
4963
4964 *is_enabled = vport->info.ipsec_packet_enabled;
4965 unlock:
4966 mutex_unlock(&esw->state_lock);
4967 return err;
4968 }
4969
mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4970 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4971 bool enable,
4972 struct netlink_ext_ack *extack)
4973 {
4974 struct mlx5_eswitch *esw;
4975 struct mlx5_vport *vport;
4976 u16 vport_num;
4977 int err;
4978
4979 esw = mlx5_devlink_eswitch_get(port->devlink);
4980 if (IS_ERR(esw))
4981 return PTR_ERR(esw);
4982
4983 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4984 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num);
4985 if (err) {
4986 NL_SET_ERR_MSG_MOD(extack,
4987 "Device doesn't support IPsec packet mode");
4988 return err;
4989 }
4990
4991 vport = mlx5_devlink_port_vport_get(port);
4992 mutex_lock(&esw->state_lock);
4993 if (!vport->enabled) {
4994 err = -EOPNOTSUPP;
4995 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4996 goto unlock;
4997 }
4998
4999 if (vport->info.ipsec_packet_enabled == enable)
5000 goto unlock;
5001
5002 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
5003 err = -EBUSY;
5004 goto unlock;
5005 }
5006
5007 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
5008 if (err) {
5009 NL_SET_ERR_MSG_MOD(extack,
5010 "Failed to set IPsec packet mode");
5011 goto unlock;
5012 }
5013
5014 vport->info.ipsec_packet_enabled = enable;
5015 if (enable)
5016 esw->enabled_ipsec_vf_count++;
5017 else
5018 esw->enabled_ipsec_vf_count--;
5019 unlock:
5020 mutex_unlock(&esw->state_lock);
5021 return err;
5022 }
5023 #endif /* CONFIG_XFRM_OFFLOAD */
5024
5025 int
mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port * port,u32 * max_io_eqs,struct netlink_ext_ack * extack)5026 mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs,
5027 struct netlink_ext_ack *extack)
5028 {
5029 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
5030 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
5031 u16 vport_num = vport->vport;
5032 struct mlx5_eswitch *esw;
5033 void *query_ctx;
5034 void *hca_caps;
5035 u32 max_eqs;
5036 int err;
5037
5038 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
5039 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
5040 NL_SET_ERR_MSG_MOD(extack,
5041 "Device doesn't support VHCA management");
5042 return -EOPNOTSUPP;
5043 }
5044
5045 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) {
5046 NL_SET_ERR_MSG_MOD(extack,
5047 "Device doesn't support getting the max number of EQs");
5048 return -EOPNOTSUPP;
5049 }
5050
5051 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
5052 if (!query_ctx)
5053 return -ENOMEM;
5054
5055 mutex_lock(&esw->state_lock);
5056 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
5057 MLX5_CAP_GENERAL_2);
5058 if (err) {
5059 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
5060 goto out;
5061 }
5062
5063 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
5064 max_eqs = MLX5_GET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b);
5065 if (max_eqs < MLX5_ESW_MAX_CTRL_EQS)
5066 *max_io_eqs = 0;
5067 else
5068 *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS;
5069 out:
5070 mutex_unlock(&esw->state_lock);
5071 kfree(query_ctx);
5072 return err;
5073 }
5074
5075 int
mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port * port,u32 max_io_eqs,struct netlink_ext_ack * extack)5076 mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs,
5077 struct netlink_ext_ack *extack)
5078 {
5079 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
5080 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
5081 u16 vport_num = vport->vport;
5082 struct mlx5_eswitch *esw;
5083 void *query_ctx;
5084 void *hca_caps;
5085 u16 max_eqs;
5086 int err;
5087
5088 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
5089 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
5090 NL_SET_ERR_MSG_MOD(extack,
5091 "Device doesn't support VHCA management");
5092 return -EOPNOTSUPP;
5093 }
5094
5095 if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) {
5096 NL_SET_ERR_MSG_MOD(extack,
5097 "Device doesn't support changing the max number of EQs");
5098 return -EOPNOTSUPP;
5099 }
5100
5101 if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) {
5102 NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range");
5103 return -EINVAL;
5104 }
5105
5106 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
5107 if (!query_ctx)
5108 return -ENOMEM;
5109
5110 mutex_lock(&esw->state_lock);
5111 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
5112 MLX5_CAP_GENERAL_2);
5113 if (err) {
5114 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
5115 goto out;
5116 }
5117
5118 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
5119 MLX5_SET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b, max_eqs);
5120
5121 if (mlx5_esw_is_sf_vport(esw, vport_num))
5122 MLX5_SET(cmd_hca_cap_2, hca_caps, sf_eq_usage, 1);
5123
5124 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
5125 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
5126 if (err)
5127 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps");
5128 vport->max_eqs_set = true;
5129 out:
5130 mutex_unlock(&esw->state_lock);
5131 kfree(query_ctx);
5132 return err;
5133 }
5134
5135 int
mlx5_devlink_port_fn_max_io_eqs_set_sf_default(struct devlink_port * port,struct netlink_ext_ack * extack)5136 mlx5_devlink_port_fn_max_io_eqs_set_sf_default(struct devlink_port *port,
5137 struct netlink_ext_ack *extack)
5138 {
5139 return mlx5_devlink_port_fn_max_io_eqs_set(port,
5140 MLX5_ESW_DEFAULT_SF_COMP_EQS,
5141 extack);
5142 }
5143