1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3 
4 #include "fs_core.h"
5 #include "fs_cmd.h"
6 #include "en.h"
7 #include "lib/ipsec_fs_roce.h"
8 #include "mlx5_core.h"
9 #include <linux/random.h>
10 
11 struct mlx5_ipsec_miss {
12 	struct mlx5_flow_group *group;
13 	struct mlx5_flow_handle *rule;
14 };
15 
16 struct mlx5_ipsec_rx_roce {
17 	struct mlx5_flow_group *g;
18 	struct mlx5_flow_table *ft;
19 	struct mlx5_flow_handle *rule;
20 	struct mlx5_ipsec_miss roce_miss;
21 	struct mlx5_flow_table *nic_master_ft;
22 	struct mlx5_flow_group *nic_master_group;
23 	struct mlx5_flow_handle *nic_master_rule;
24 	struct mlx5_flow_table *goto_alias_ft;
25 	u32 alias_id;
26 	char key[ACCESS_KEY_LEN];
27 
28 	struct mlx5_flow_table *ft_rdma;
29 	struct mlx5_flow_namespace *ns_rdma;
30 };
31 
32 struct mlx5_ipsec_tx_roce {
33 	struct mlx5_flow_group *g;
34 	struct mlx5_flow_table *ft;
35 	struct mlx5_flow_handle *rule;
36 	struct mlx5_flow_table *goto_alias_ft;
37 	u32 alias_id;
38 	char key[ACCESS_KEY_LEN];
39 	struct mlx5_flow_namespace *ns;
40 };
41 
42 struct mlx5_ipsec_fs {
43 	struct mlx5_ipsec_rx_roce ipv4_rx;
44 	struct mlx5_ipsec_rx_roce ipv6_rx;
45 	struct mlx5_ipsec_tx_roce tx;
46 	struct mlx5_devcom_comp_dev **devcom;
47 };
48 
ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec * spec,u16 dport)49 static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec,
50 					  u16 dport)
51 {
52 	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
53 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
54 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
55 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
56 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
57 }
58 
ipsec_fs_create_alias_supported_one(struct mlx5_core_dev * mdev)59 static bool ipsec_fs_create_alias_supported_one(struct mlx5_core_dev *mdev)
60 {
61 	u64 obj_allowed = MLX5_CAP_GEN_2_64(mdev, allowed_object_for_other_vhca_access);
62 	u32 obj_supp = MLX5_CAP_GEN_2(mdev, cross_vhca_object_to_object_supported);
63 
64 	if (!(obj_supp &
65 	    MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS))
66 		return false;
67 
68 	if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
69 		return false;
70 
71 	return true;
72 }
73 
ipsec_fs_create_alias_supported(struct mlx5_core_dev * mdev,struct mlx5_core_dev * master_mdev)74 static bool ipsec_fs_create_alias_supported(struct mlx5_core_dev *mdev,
75 					    struct mlx5_core_dev *master_mdev)
76 {
77 	if (ipsec_fs_create_alias_supported_one(mdev) &&
78 	    ipsec_fs_create_alias_supported_one(master_mdev))
79 		return true;
80 
81 	return false;
82 }
83 
ipsec_fs_create_aliased_ft(struct mlx5_core_dev * ibv_owner,struct mlx5_core_dev * ibv_allowed,struct mlx5_flow_table * ft,u32 * obj_id,char * alias_key,bool from_event)84 static int ipsec_fs_create_aliased_ft(struct mlx5_core_dev *ibv_owner,
85 				      struct mlx5_core_dev *ibv_allowed,
86 				      struct mlx5_flow_table *ft,
87 				      u32 *obj_id, char *alias_key, bool from_event)
88 {
89 	u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
90 	u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(ibv_owner, vhca_id);
91 	struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
92 	struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
93 	int ret;
94 	int i;
95 
96 	if (!ipsec_fs_create_alias_supported(ibv_owner, ibv_allowed))
97 		return -EOPNOTSUPP;
98 
99 	for (i = 0; i < ACCESS_KEY_LEN; i++)
100 		if (!from_event)
101 			alias_key[i] = get_random_u64() & 0xFF;
102 
103 	memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
104 	allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
105 	allow_attr.obj_id = aliased_object_id;
106 
107 	if (!from_event) {
108 		ret = mlx5_cmd_allow_other_vhca_access(ibv_owner, &allow_attr);
109 		if (ret) {
110 			mlx5_core_err(ibv_owner, "Failed to allow other vhca access err=%d\n",
111 				      ret);
112 			return ret;
113 		}
114 	}
115 
116 	memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
117 	alias_attr.obj_id = aliased_object_id;
118 	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
119 	alias_attr.vhca_id = vhca_id_to_be_accessed;
120 	ret = mlx5_cmd_alias_obj_create(ibv_allowed, &alias_attr, obj_id);
121 	if (ret) {
122 		mlx5_core_err(ibv_allowed, "Failed to create alias object err=%d\n",
123 			      ret);
124 		return ret;
125 	}
126 
127 	return 0;
128 }
129 
130 static int
ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_flow_destination * default_dst,struct mlx5_ipsec_rx_roce * roce)131 ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev,
132 			    struct mlx5_flow_destination *default_dst,
133 			    struct mlx5_ipsec_rx_roce *roce)
134 {
135 	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
136 	struct mlx5_flow_destination dst = {};
137 	MLX5_DECLARE_FLOW_ACT(flow_act);
138 	struct mlx5_flow_handle *rule;
139 	struct mlx5_flow_spec *spec;
140 	int err = 0;
141 
142 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
143 	if (!spec)
144 		return -ENOMEM;
145 
146 	ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
147 
148 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
149 	if (is_mpv_slave) {
150 		dst.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
151 		dst.ft = roce->goto_alias_ft;
152 	} else {
153 		dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
154 		dst.ft = roce->ft_rdma;
155 	}
156 	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
157 	if (IS_ERR(rule)) {
158 		err = PTR_ERR(rule);
159 		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n",
160 			      err);
161 		goto out;
162 	}
163 
164 	roce->rule = rule;
165 
166 	memset(spec, 0, sizeof(*spec));
167 	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1);
168 	if (IS_ERR(rule)) {
169 		err = PTR_ERR(rule);
170 		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n",
171 			      err);
172 		goto fail_add_default_rule;
173 	}
174 
175 	roce->roce_miss.rule = rule;
176 
177 	if (!is_mpv_slave)
178 		goto out;
179 
180 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
181 	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
182 	dst.ft = roce->ft_rdma;
183 	rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst,
184 				   1);
185 	if (IS_ERR(rule)) {
186 		err = PTR_ERR(rule);
187 		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule for alias err=%d\n",
188 			      err);
189 		goto fail_add_nic_master_rule;
190 	}
191 	roce->nic_master_rule = rule;
192 
193 	kvfree(spec);
194 	return 0;
195 
196 fail_add_nic_master_rule:
197 	mlx5_del_flow_rules(roce->roce_miss.rule);
198 fail_add_default_rule:
199 	mlx5_del_flow_rules(roce->rule);
200 out:
201 	kvfree(spec);
202 	return err;
203 }
204 
ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft)205 static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev,
206 				       struct mlx5_ipsec_tx_roce *roce,
207 				       struct mlx5_flow_table *pol_ft)
208 {
209 	struct mlx5_flow_destination dst = {};
210 	MLX5_DECLARE_FLOW_ACT(flow_act);
211 	struct mlx5_flow_handle *rule;
212 	int err = 0;
213 
214 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
215 	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
216 	dst.ft = pol_ft;
217 	rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst,
218 				   1);
219 	if (IS_ERR(rule)) {
220 		err = PTR_ERR(rule);
221 		mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
222 			      err);
223 		goto out;
224 	}
225 	roce->rule = rule;
226 
227 out:
228 	return err;
229 }
230 
ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft)231 static int ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev *mdev,
232 					   struct mlx5_ipsec_tx_roce *roce,
233 					   struct mlx5_flow_table *pol_ft)
234 {
235 	struct mlx5_flow_destination dst = {};
236 	MLX5_DECLARE_FLOW_ACT(flow_act);
237 	struct mlx5_flow_handle *rule;
238 	struct mlx5_flow_spec *spec;
239 	int err = 0;
240 
241 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
242 	if (!spec)
243 		return -ENOMEM;
244 
245 	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
246 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.source_vhca_port);
247 	MLX5_SET(fte_match_param, spec->match_value, misc_parameters.source_vhca_port,
248 		 MLX5_CAP_GEN(mdev, native_port_num));
249 
250 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
251 	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
252 	dst.ft = roce->goto_alias_ft;
253 	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
254 	if (IS_ERR(rule)) {
255 		err = PTR_ERR(rule);
256 		mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
257 			      err);
258 		goto out;
259 	}
260 	roce->rule = rule;
261 
262 	/* No need for miss rule, since on miss we go to next PRIO, in which
263 	 * if master is configured, he will catch the traffic to go to his
264 	 * encryption table.
265 	 */
266 
267 out:
268 	kvfree(spec);
269 	return err;
270 }
271 
272 #define MLX5_TX_ROCE_GROUP_SIZE BIT(0)
273 #define MLX5_IPSEC_RDMA_TX_FT_LEVEL 0
274 #define MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL 3 /* Since last used level in NIC ipsec is 2 */
275 
ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft,struct mlx5e_priv * peer_priv,bool from_event)276 static int ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev *mdev,
277 					  struct mlx5_ipsec_tx_roce *roce,
278 					  struct mlx5_flow_table *pol_ft,
279 					  struct mlx5e_priv *peer_priv,
280 					  bool from_event)
281 {
282 	struct mlx5_flow_namespace *roce_ns, *nic_ns;
283 	struct mlx5_flow_table_attr ft_attr = {};
284 	struct mlx5_flow_table next_ft;
285 	struct mlx5_flow_table *ft;
286 	int err;
287 
288 	roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
289 	if (!roce_ns)
290 		return -EOPNOTSUPP;
291 
292 	nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
293 	if (!nic_ns)
294 		return -EOPNOTSUPP;
295 
296 	err = ipsec_fs_create_aliased_ft(mdev, peer_priv->mdev, pol_ft, &roce->alias_id, roce->key,
297 					 from_event);
298 	if (err)
299 		return err;
300 
301 	next_ft.id = roce->alias_id;
302 	ft_attr.max_fte = 1;
303 	ft_attr.next_ft = &next_ft;
304 	ft_attr.level = MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL;
305 	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
306 	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
307 	if (IS_ERR(ft)) {
308 		err = PTR_ERR(ft);
309 		mlx5_core_err(mdev, "Fail to create RoCE IPsec goto alias ft err=%d\n", err);
310 		goto destroy_alias;
311 	}
312 
313 	roce->goto_alias_ft = ft;
314 
315 	memset(&ft_attr, 0, sizeof(ft_attr));
316 	ft_attr.max_fte = 1;
317 	ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
318 	ft = mlx5_create_flow_table(roce_ns, &ft_attr);
319 	if (IS_ERR(ft)) {
320 		err = PTR_ERR(ft);
321 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
322 		goto destroy_alias_ft;
323 	}
324 
325 	roce->ft = ft;
326 
327 	return 0;
328 
329 destroy_alias_ft:
330 	mlx5_destroy_flow_table(roce->goto_alias_ft);
331 destroy_alias:
332 	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
333 				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
334 	return err;
335 }
336 
ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft,u32 * in)337 static int ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev *mdev,
338 						   struct mlx5_ipsec_tx_roce *roce,
339 						   struct mlx5_flow_table *pol_ft,
340 						   u32 *in)
341 {
342 	struct mlx5_flow_group *g;
343 	int ix = 0;
344 	int err;
345 	u8 *mc;
346 
347 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
348 	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters.source_vhca_port);
349 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS);
350 
351 	MLX5_SET_CFG(in, start_flow_index, ix);
352 	ix += MLX5_TX_ROCE_GROUP_SIZE;
353 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
354 	g = mlx5_create_flow_group(roce->ft, in);
355 	if (IS_ERR(g)) {
356 		err = PTR_ERR(g);
357 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
358 		return err;
359 	}
360 	roce->g = g;
361 
362 	err = ipsec_fs_roce_tx_mpv_rule_setup(mdev, roce, pol_ft);
363 	if (err) {
364 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
365 		goto destroy_group;
366 	}
367 
368 	return 0;
369 
370 destroy_group:
371 	mlx5_destroy_flow_group(roce->g);
372 	return err;
373 }
374 
ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_table * pol_ft,u32 * in,bool from_event)375 static int ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev *mdev,
376 				       struct mlx5_ipsec_fs *ipsec_roce,
377 				       struct mlx5_flow_table *pol_ft,
378 				       u32 *in, bool from_event)
379 {
380 	struct mlx5_devcom_comp_dev *tmp = NULL;
381 	struct mlx5_ipsec_tx_roce *roce;
382 	struct mlx5e_priv *peer_priv;
383 	int err;
384 
385 	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
386 		return -EOPNOTSUPP;
387 
388 	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
389 	if (!peer_priv) {
390 		err = -EOPNOTSUPP;
391 		goto release_peer;
392 	}
393 
394 	roce = &ipsec_roce->tx;
395 
396 	err = ipsec_fs_roce_tx_mpv_create_ft(mdev, roce, pol_ft, peer_priv, from_event);
397 	if (err) {
398 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tables err=%d\n", err);
399 		goto release_peer;
400 	}
401 
402 	err = ipsec_fs_roce_tx_mpv_create_group_rules(mdev, roce, pol_ft, in);
403 	if (err) {
404 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group/rule err=%d\n", err);
405 		goto destroy_tables;
406 	}
407 
408 	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
409 	return 0;
410 
411 destroy_tables:
412 	mlx5_destroy_flow_table(roce->ft);
413 	mlx5_destroy_flow_table(roce->goto_alias_ft);
414 	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
415 				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
416 release_peer:
417 	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
418 	return err;
419 }
420 
roce_rx_mpv_destroy_tables(struct mlx5_core_dev * mdev,struct mlx5_ipsec_rx_roce * roce)421 static void roce_rx_mpv_destroy_tables(struct mlx5_core_dev *mdev, struct mlx5_ipsec_rx_roce *roce)
422 {
423 	mlx5_destroy_flow_table(roce->goto_alias_ft);
424 	mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
425 				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
426 	mlx5_destroy_flow_group(roce->nic_master_group);
427 	mlx5_destroy_flow_table(roce->nic_master_ft);
428 }
429 
430 #define MLX5_RX_ROCE_GROUP_SIZE BIT(0)
431 #define MLX5_IPSEC_RX_IPV4_FT_LEVEL 3
432 #define MLX5_IPSEC_RX_IPV6_FT_LEVEL 2
433 
ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_namespace * ns,u32 family,u32 level,u32 prio)434 static int ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev *mdev,
435 				       struct mlx5_ipsec_fs *ipsec_roce,
436 				       struct mlx5_flow_namespace *ns,
437 				       u32 family, u32 level, u32 prio)
438 {
439 	struct mlx5_flow_namespace *roce_ns, *nic_ns;
440 	struct mlx5_flow_table_attr ft_attr = {};
441 	struct mlx5_devcom_comp_dev *tmp = NULL;
442 	struct mlx5_ipsec_rx_roce *roce;
443 	struct mlx5_flow_table next_ft;
444 	struct mlx5_flow_table *ft;
445 	struct mlx5_flow_group *g;
446 	struct mlx5e_priv *peer_priv;
447 	int ix = 0;
448 	u32 *in;
449 	int err;
450 
451 	roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
452 				     &ipsec_roce->ipv6_rx;
453 
454 	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
455 		return -EOPNOTSUPP;
456 
457 	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
458 	if (!peer_priv) {
459 		err = -EOPNOTSUPP;
460 		goto release_peer;
461 	}
462 
463 	roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
464 	if (!roce_ns) {
465 		err = -EOPNOTSUPP;
466 		goto release_peer;
467 	}
468 
469 	nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
470 	if (!nic_ns) {
471 		err = -EOPNOTSUPP;
472 		goto release_peer;
473 	}
474 
475 	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
476 	if (!in) {
477 		err = -ENOMEM;
478 		goto release_peer;
479 	}
480 
481 	ft_attr.level = (family == AF_INET) ? MLX5_IPSEC_RX_IPV4_FT_LEVEL :
482 					      MLX5_IPSEC_RX_IPV6_FT_LEVEL;
483 	ft_attr.max_fte = 1;
484 	ft = mlx5_create_flow_table(roce_ns, &ft_attr);
485 	if (IS_ERR(ft)) {
486 		err = PTR_ERR(ft);
487 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma master err=%d\n", err);
488 		goto free_in;
489 	}
490 
491 	roce->ft_rdma = ft;
492 
493 	ft_attr.max_fte = 1;
494 	ft_attr.prio = prio;
495 	ft_attr.level = level + 2;
496 	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
497 	if (IS_ERR(ft)) {
498 		err = PTR_ERR(ft);
499 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC master err=%d\n", err);
500 		goto destroy_ft_rdma;
501 	}
502 	roce->nic_master_ft = ft;
503 
504 	MLX5_SET_CFG(in, start_flow_index, ix);
505 	ix += 1;
506 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
507 	g = mlx5_create_flow_group(roce->nic_master_ft, in);
508 	if (IS_ERR(g)) {
509 		err = PTR_ERR(g);
510 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group aliased err=%d\n", err);
511 		goto destroy_nic_master_ft;
512 	}
513 	roce->nic_master_group = g;
514 
515 	err = ipsec_fs_create_aliased_ft(peer_priv->mdev, mdev, roce->nic_master_ft,
516 					 &roce->alias_id, roce->key, false);
517 	if (err) {
518 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias FT err=%d\n", err);
519 		goto destroy_group;
520 	}
521 
522 	next_ft.id = roce->alias_id;
523 	ft_attr.max_fte = 1;
524 	ft_attr.prio = prio;
525 	ft_attr.level = roce->ft->level + 1;
526 	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
527 	ft_attr.next_ft = &next_ft;
528 	ft = mlx5_create_flow_table(ns, &ft_attr);
529 	if (IS_ERR(ft)) {
530 		err = PTR_ERR(ft);
531 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC slave err=%d\n", err);
532 		goto destroy_alias;
533 	}
534 	roce->goto_alias_ft = ft;
535 
536 	kvfree(in);
537 	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
538 	return 0;
539 
540 destroy_alias:
541 	mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
542 				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
543 destroy_group:
544 	mlx5_destroy_flow_group(roce->nic_master_group);
545 destroy_nic_master_ft:
546 	mlx5_destroy_flow_table(roce->nic_master_ft);
547 destroy_ft_rdma:
548 	mlx5_destroy_flow_table(roce->ft_rdma);
549 free_in:
550 	kvfree(in);
551 release_peer:
552 	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
553 	return err;
554 }
555 
mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_core_dev * mdev)556 void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce,
557 				   struct mlx5_core_dev *mdev)
558 {
559 	struct mlx5_devcom_comp_dev *tmp = NULL;
560 	struct mlx5_ipsec_tx_roce *tx_roce;
561 	struct mlx5e_priv *peer_priv;
562 
563 	if (!ipsec_roce)
564 		return;
565 
566 	tx_roce = &ipsec_roce->tx;
567 
568 	if (!tx_roce->ft)
569 		return; /* Incase RoCE was cleaned from MPV event flow */
570 
571 	mlx5_del_flow_rules(tx_roce->rule);
572 	mlx5_destroy_flow_group(tx_roce->g);
573 	mlx5_destroy_flow_table(tx_roce->ft);
574 
575 	if (!mlx5_core_is_mp_slave(mdev))
576 		return;
577 
578 	if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
579 		return;
580 
581 	peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
582 	if (!peer_priv) {
583 		mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
584 		return;
585 	}
586 
587 	mlx5_destroy_flow_table(tx_roce->goto_alias_ft);
588 	mlx5_cmd_alias_obj_destroy(peer_priv->mdev, tx_roce->alias_id,
589 				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
590 	mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
591 	tx_roce->ft = NULL;
592 }
593 
mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_table * pol_ft,bool from_event)594 int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev,
595 				 struct mlx5_ipsec_fs *ipsec_roce,
596 				 struct mlx5_flow_table *pol_ft,
597 				 bool from_event)
598 {
599 	struct mlx5_flow_table_attr ft_attr = {};
600 	struct mlx5_ipsec_tx_roce *roce;
601 	struct mlx5_flow_table *ft;
602 	struct mlx5_flow_group *g;
603 	int ix = 0;
604 	int err;
605 	u32 *in;
606 
607 	if (!ipsec_roce)
608 		return 0;
609 
610 	roce = &ipsec_roce->tx;
611 
612 	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
613 	if (!in)
614 		return -ENOMEM;
615 
616 	if (mlx5_core_is_mp_slave(mdev)) {
617 		err = ipsec_fs_roce_tx_mpv_create(mdev, ipsec_roce, pol_ft, in, from_event);
618 		goto free_in;
619 	}
620 
621 	ft_attr.max_fte = 1;
622 	ft_attr.prio = 1;
623 	ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
624 	ft = mlx5_create_flow_table(roce->ns, &ft_attr);
625 	if (IS_ERR(ft)) {
626 		err = PTR_ERR(ft);
627 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
628 		goto free_in;
629 	}
630 
631 	roce->ft = ft;
632 
633 	MLX5_SET_CFG(in, start_flow_index, ix);
634 	ix += MLX5_TX_ROCE_GROUP_SIZE;
635 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
636 	g = mlx5_create_flow_group(ft, in);
637 	if (IS_ERR(g)) {
638 		err = PTR_ERR(g);
639 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
640 		goto destroy_table;
641 	}
642 	roce->g = g;
643 
644 	err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft);
645 	if (err) {
646 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
647 		goto destroy_group;
648 	}
649 
650 	kvfree(in);
651 	return 0;
652 
653 destroy_group:
654 	mlx5_destroy_flow_group(roce->g);
655 destroy_table:
656 	mlx5_destroy_flow_table(ft);
657 free_in:
658 	kvfree(in);
659 	return err;
660 }
661 
mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs * ipsec_roce,u32 family)662 struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family)
663 {
664 	struct mlx5_ipsec_rx_roce *rx_roce;
665 
666 	if (!ipsec_roce)
667 		return NULL;
668 
669 	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
670 					&ipsec_roce->ipv6_rx;
671 
672 	return rx_roce->ft;
673 }
674 
mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs * ipsec_roce,u32 family,struct mlx5_core_dev * mdev)675 void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family,
676 				   struct mlx5_core_dev *mdev)
677 {
678 	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
679 	struct mlx5_ipsec_rx_roce *rx_roce;
680 
681 	if (!ipsec_roce)
682 		return;
683 
684 	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
685 					&ipsec_roce->ipv6_rx;
686 	if (!rx_roce->ft)
687 		return; /* Incase RoCE was cleaned from MPV event flow */
688 
689 	if (is_mpv_slave)
690 		mlx5_del_flow_rules(rx_roce->nic_master_rule);
691 	mlx5_del_flow_rules(rx_roce->roce_miss.rule);
692 	mlx5_del_flow_rules(rx_roce->rule);
693 	if (is_mpv_slave)
694 		roce_rx_mpv_destroy_tables(mdev, rx_roce);
695 	mlx5_destroy_flow_table(rx_roce->ft_rdma);
696 	mlx5_destroy_flow_group(rx_roce->roce_miss.group);
697 	mlx5_destroy_flow_group(rx_roce->g);
698 	mlx5_destroy_flow_table(rx_roce->ft);
699 	rx_roce->ft = NULL;
700 }
701 
mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_namespace * ns,struct mlx5_flow_destination * default_dst,u32 family,u32 level,u32 prio)702 int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev,
703 				 struct mlx5_ipsec_fs *ipsec_roce,
704 				 struct mlx5_flow_namespace *ns,
705 				 struct mlx5_flow_destination *default_dst,
706 				 u32 family, u32 level, u32 prio)
707 {
708 	bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
709 	struct mlx5_flow_table_attr ft_attr = {};
710 	struct mlx5_ipsec_rx_roce *roce;
711 	struct mlx5_flow_table *ft;
712 	struct mlx5_flow_group *g;
713 	void *outer_headers_c;
714 	int ix = 0;
715 	u32 *in;
716 	int err;
717 	u8 *mc;
718 
719 	if (!ipsec_roce)
720 		return 0;
721 
722 	roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
723 				     &ipsec_roce->ipv6_rx;
724 
725 	ft_attr.max_fte = 2;
726 	ft_attr.level = level;
727 	ft_attr.prio = prio;
728 	ft = mlx5_create_flow_table(ns, &ft_attr);
729 	if (IS_ERR(ft)) {
730 		err = PTR_ERR(ft);
731 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err);
732 		return err;
733 	}
734 
735 	roce->ft = ft;
736 
737 	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
738 	if (!in) {
739 		err = -ENOMEM;
740 		goto fail_nomem;
741 	}
742 
743 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
744 	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
745 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
746 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
747 
748 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
749 	MLX5_SET_CFG(in, start_flow_index, ix);
750 	ix += MLX5_RX_ROCE_GROUP_SIZE;
751 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
752 	g = mlx5_create_flow_group(ft, in);
753 	if (IS_ERR(g)) {
754 		err = PTR_ERR(g);
755 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err);
756 		goto fail_group;
757 	}
758 	roce->g = g;
759 
760 	memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
761 	MLX5_SET_CFG(in, start_flow_index, ix);
762 	ix += MLX5_RX_ROCE_GROUP_SIZE;
763 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
764 	g = mlx5_create_flow_group(ft, in);
765 	if (IS_ERR(g)) {
766 		err = PTR_ERR(g);
767 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err);
768 		goto fail_mgroup;
769 	}
770 	roce->roce_miss.group = g;
771 
772 	if (is_mpv_slave) {
773 		err = ipsec_fs_roce_rx_mpv_create(mdev, ipsec_roce, ns, family, level, prio);
774 		if (err) {
775 			mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias err=%d\n", err);
776 			goto fail_mpv_create;
777 		}
778 	} else {
779 		memset(&ft_attr, 0, sizeof(ft_attr));
780 		if (family == AF_INET)
781 			ft_attr.level = 1;
782 		ft_attr.max_fte = 1;
783 		ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr);
784 		if (IS_ERR(ft)) {
785 			err = PTR_ERR(ft);
786 			mlx5_core_err(mdev,
787 				      "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err);
788 			goto fail_rdma_table;
789 		}
790 
791 		roce->ft_rdma = ft;
792 	}
793 
794 	err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce);
795 	if (err) {
796 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err);
797 		goto fail_setup_rule;
798 	}
799 
800 	kvfree(in);
801 	return 0;
802 
803 fail_setup_rule:
804 	if (is_mpv_slave)
805 		roce_rx_mpv_destroy_tables(mdev, roce);
806 	mlx5_destroy_flow_table(roce->ft_rdma);
807 fail_mpv_create:
808 fail_rdma_table:
809 	mlx5_destroy_flow_group(roce->roce_miss.group);
810 fail_mgroup:
811 	mlx5_destroy_flow_group(roce->g);
812 fail_group:
813 	kvfree(in);
814 fail_nomem:
815 	mlx5_destroy_flow_table(roce->ft);
816 	return err;
817 }
818 
mlx5_ipsec_fs_is_mpv_roce_supported(struct mlx5_core_dev * mdev)819 bool mlx5_ipsec_fs_is_mpv_roce_supported(struct mlx5_core_dev *mdev)
820 {
821 	if (!mlx5_core_mp_enabled(mdev))
822 		return true;
823 
824 	if (ipsec_fs_create_alias_supported_one(mdev))
825 		return true;
826 
827 	return false;
828 }
829 
mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs * ipsec_roce)830 void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce)
831 {
832 	kfree(ipsec_roce);
833 }
834 
mlx5_ipsec_fs_roce_init(struct mlx5_core_dev * mdev,struct mlx5_devcom_comp_dev ** devcom)835 struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev,
836 					      struct mlx5_devcom_comp_dev **devcom)
837 {
838 	struct mlx5_ipsec_fs *roce_ipsec;
839 	struct mlx5_flow_namespace *ns;
840 
841 	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
842 	if (!ns) {
843 		mlx5_core_err(mdev, "Failed to get RoCE rx ns\n");
844 		return NULL;
845 	}
846 
847 	roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL);
848 	if (!roce_ipsec)
849 		return NULL;
850 
851 	roce_ipsec->ipv4_rx.ns_rdma = ns;
852 	roce_ipsec->ipv6_rx.ns_rdma = ns;
853 
854 	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
855 	if (!ns) {
856 		mlx5_core_err(mdev, "Failed to get RoCE tx ns\n");
857 		goto err_tx;
858 	}
859 
860 	roce_ipsec->tx.ns = ns;
861 
862 	roce_ipsec->devcom = devcom;
863 
864 	return roce_ipsec;
865 
866 err_tx:
867 	kfree(roce_ipsec);
868 	return NULL;
869 }
870