1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3
4 #include "fs_core.h"
5 #include "fs_cmd.h"
6 #include "en.h"
7 #include "lib/ipsec_fs_roce.h"
8 #include "mlx5_core.h"
9 #include <linux/random.h>
10
11 struct mlx5_ipsec_miss {
12 struct mlx5_flow_group *group;
13 struct mlx5_flow_handle *rule;
14 };
15
16 struct mlx5_ipsec_rx_roce {
17 struct mlx5_flow_group *g;
18 struct mlx5_flow_table *ft;
19 struct mlx5_flow_handle *rule;
20 struct mlx5_ipsec_miss roce_miss;
21 struct mlx5_flow_table *nic_master_ft;
22 struct mlx5_flow_group *nic_master_group;
23 struct mlx5_flow_handle *nic_master_rule;
24 struct mlx5_flow_table *goto_alias_ft;
25 u32 alias_id;
26 char key[ACCESS_KEY_LEN];
27
28 struct mlx5_flow_table *ft_rdma;
29 struct mlx5_flow_namespace *ns_rdma;
30 };
31
32 struct mlx5_ipsec_tx_roce {
33 struct mlx5_flow_group *g;
34 struct mlx5_flow_table *ft;
35 struct mlx5_flow_handle *rule;
36 struct mlx5_flow_table *goto_alias_ft;
37 u32 alias_id;
38 char key[ACCESS_KEY_LEN];
39 struct mlx5_flow_namespace *ns;
40 };
41
42 struct mlx5_ipsec_fs {
43 struct mlx5_ipsec_rx_roce ipv4_rx;
44 struct mlx5_ipsec_rx_roce ipv6_rx;
45 struct mlx5_ipsec_tx_roce tx;
46 struct mlx5_devcom_comp_dev **devcom;
47 };
48
ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec * spec,u16 dport)49 static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec,
50 u16 dport)
51 {
52 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
54 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
56 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
57 }
58
ipsec_fs_create_alias_supported_one(struct mlx5_core_dev * mdev)59 static bool ipsec_fs_create_alias_supported_one(struct mlx5_core_dev *mdev)
60 {
61 u64 obj_allowed = MLX5_CAP_GEN_2_64(mdev, allowed_object_for_other_vhca_access);
62 u32 obj_supp = MLX5_CAP_GEN_2(mdev, cross_vhca_object_to_object_supported);
63
64 if (!(obj_supp &
65 MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS))
66 return false;
67
68 if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
69 return false;
70
71 return true;
72 }
73
ipsec_fs_create_alias_supported(struct mlx5_core_dev * mdev,struct mlx5_core_dev * master_mdev)74 static bool ipsec_fs_create_alias_supported(struct mlx5_core_dev *mdev,
75 struct mlx5_core_dev *master_mdev)
76 {
77 if (ipsec_fs_create_alias_supported_one(mdev) &&
78 ipsec_fs_create_alias_supported_one(master_mdev))
79 return true;
80
81 return false;
82 }
83
ipsec_fs_create_aliased_ft(struct mlx5_core_dev * ibv_owner,struct mlx5_core_dev * ibv_allowed,struct mlx5_flow_table * ft,u32 * obj_id,char * alias_key,bool from_event)84 static int ipsec_fs_create_aliased_ft(struct mlx5_core_dev *ibv_owner,
85 struct mlx5_core_dev *ibv_allowed,
86 struct mlx5_flow_table *ft,
87 u32 *obj_id, char *alias_key, bool from_event)
88 {
89 u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
90 u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(ibv_owner, vhca_id);
91 struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
92 struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
93 int ret;
94 int i;
95
96 if (!ipsec_fs_create_alias_supported(ibv_owner, ibv_allowed))
97 return -EOPNOTSUPP;
98
99 for (i = 0; i < ACCESS_KEY_LEN; i++)
100 if (!from_event)
101 alias_key[i] = get_random_u64() & 0xFF;
102
103 memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
104 allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
105 allow_attr.obj_id = aliased_object_id;
106
107 if (!from_event) {
108 ret = mlx5_cmd_allow_other_vhca_access(ibv_owner, &allow_attr);
109 if (ret) {
110 mlx5_core_err(ibv_owner, "Failed to allow other vhca access err=%d\n",
111 ret);
112 return ret;
113 }
114 }
115
116 memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
117 alias_attr.obj_id = aliased_object_id;
118 alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
119 alias_attr.vhca_id = vhca_id_to_be_accessed;
120 ret = mlx5_cmd_alias_obj_create(ibv_allowed, &alias_attr, obj_id);
121 if (ret) {
122 mlx5_core_err(ibv_allowed, "Failed to create alias object err=%d\n",
123 ret);
124 return ret;
125 }
126
127 return 0;
128 }
129
130 static int
ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_flow_destination * default_dst,struct mlx5_ipsec_rx_roce * roce)131 ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev,
132 struct mlx5_flow_destination *default_dst,
133 struct mlx5_ipsec_rx_roce *roce)
134 {
135 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
136 struct mlx5_flow_destination dst = {};
137 MLX5_DECLARE_FLOW_ACT(flow_act);
138 struct mlx5_flow_handle *rule;
139 struct mlx5_flow_spec *spec;
140 int err = 0;
141
142 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
143 if (!spec)
144 return -ENOMEM;
145
146 ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
147
148 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
149 if (is_mpv_slave) {
150 dst.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
151 dst.ft = roce->goto_alias_ft;
152 } else {
153 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
154 dst.ft = roce->ft_rdma;
155 }
156 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
157 if (IS_ERR(rule)) {
158 err = PTR_ERR(rule);
159 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n",
160 err);
161 goto out;
162 }
163
164 roce->rule = rule;
165
166 memset(spec, 0, sizeof(*spec));
167 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1);
168 if (IS_ERR(rule)) {
169 err = PTR_ERR(rule);
170 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n",
171 err);
172 goto fail_add_default_rule;
173 }
174
175 roce->roce_miss.rule = rule;
176
177 if (!is_mpv_slave)
178 goto out;
179
180 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
181 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
182 dst.ft = roce->ft_rdma;
183 rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst,
184 1);
185 if (IS_ERR(rule)) {
186 err = PTR_ERR(rule);
187 mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule for alias err=%d\n",
188 err);
189 goto fail_add_nic_master_rule;
190 }
191 roce->nic_master_rule = rule;
192
193 kvfree(spec);
194 return 0;
195
196 fail_add_nic_master_rule:
197 mlx5_del_flow_rules(roce->roce_miss.rule);
198 fail_add_default_rule:
199 mlx5_del_flow_rules(roce->rule);
200 out:
201 kvfree(spec);
202 return err;
203 }
204
ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft)205 static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev,
206 struct mlx5_ipsec_tx_roce *roce,
207 struct mlx5_flow_table *pol_ft)
208 {
209 struct mlx5_flow_destination dst = {};
210 MLX5_DECLARE_FLOW_ACT(flow_act);
211 struct mlx5_flow_handle *rule;
212 int err = 0;
213
214 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
215 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
216 dst.ft = pol_ft;
217 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst,
218 1);
219 if (IS_ERR(rule)) {
220 err = PTR_ERR(rule);
221 mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
222 err);
223 goto out;
224 }
225 roce->rule = rule;
226
227 out:
228 return err;
229 }
230
ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft)231 static int ipsec_fs_roce_tx_mpv_rule_setup(struct mlx5_core_dev *mdev,
232 struct mlx5_ipsec_tx_roce *roce,
233 struct mlx5_flow_table *pol_ft)
234 {
235 struct mlx5_flow_destination dst = {};
236 MLX5_DECLARE_FLOW_ACT(flow_act);
237 struct mlx5_flow_handle *rule;
238 struct mlx5_flow_spec *spec;
239 int err = 0;
240
241 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
242 if (!spec)
243 return -ENOMEM;
244
245 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
246 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.source_vhca_port);
247 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.source_vhca_port,
248 MLX5_CAP_GEN(mdev, native_port_num));
249
250 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
251 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
252 dst.ft = roce->goto_alias_ft;
253 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
254 if (IS_ERR(rule)) {
255 err = PTR_ERR(rule);
256 mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
257 err);
258 goto out;
259 }
260 roce->rule = rule;
261
262 /* No need for miss rule, since on miss we go to next PRIO, in which
263 * if master is configured, he will catch the traffic to go to his
264 * encryption table.
265 */
266
267 out:
268 kvfree(spec);
269 return err;
270 }
271
272 #define MLX5_TX_ROCE_GROUP_SIZE BIT(0)
273 #define MLX5_IPSEC_RDMA_TX_FT_LEVEL 0
274 #define MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL 3 /* Since last used level in NIC ipsec is 2 */
275
ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft,struct mlx5e_priv * peer_priv,bool from_event)276 static int ipsec_fs_roce_tx_mpv_create_ft(struct mlx5_core_dev *mdev,
277 struct mlx5_ipsec_tx_roce *roce,
278 struct mlx5_flow_table *pol_ft,
279 struct mlx5e_priv *peer_priv,
280 bool from_event)
281 {
282 struct mlx5_flow_namespace *roce_ns, *nic_ns;
283 struct mlx5_flow_table_attr ft_attr = {};
284 struct mlx5_flow_table next_ft;
285 struct mlx5_flow_table *ft;
286 int err;
287
288 roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
289 if (!roce_ns)
290 return -EOPNOTSUPP;
291
292 nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
293 if (!nic_ns)
294 return -EOPNOTSUPP;
295
296 err = ipsec_fs_create_aliased_ft(mdev, peer_priv->mdev, pol_ft, &roce->alias_id, roce->key,
297 from_event);
298 if (err)
299 return err;
300
301 next_ft.id = roce->alias_id;
302 ft_attr.max_fte = 1;
303 ft_attr.next_ft = &next_ft;
304 ft_attr.level = MLX5_IPSEC_NIC_GOTO_ALIAS_FT_LEVEL;
305 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
306 ft = mlx5_create_flow_table(nic_ns, &ft_attr);
307 if (IS_ERR(ft)) {
308 err = PTR_ERR(ft);
309 mlx5_core_err(mdev, "Fail to create RoCE IPsec goto alias ft err=%d\n", err);
310 goto destroy_alias;
311 }
312
313 roce->goto_alias_ft = ft;
314
315 memset(&ft_attr, 0, sizeof(ft_attr));
316 ft_attr.max_fte = 1;
317 ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
318 ft = mlx5_create_flow_table(roce_ns, &ft_attr);
319 if (IS_ERR(ft)) {
320 err = PTR_ERR(ft);
321 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
322 goto destroy_alias_ft;
323 }
324
325 roce->ft = ft;
326
327 return 0;
328
329 destroy_alias_ft:
330 mlx5_destroy_flow_table(roce->goto_alias_ft);
331 destroy_alias:
332 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
333 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
334 return err;
335 }
336
ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft,u32 * in)337 static int ipsec_fs_roce_tx_mpv_create_group_rules(struct mlx5_core_dev *mdev,
338 struct mlx5_ipsec_tx_roce *roce,
339 struct mlx5_flow_table *pol_ft,
340 u32 *in)
341 {
342 struct mlx5_flow_group *g;
343 int ix = 0;
344 int err;
345 u8 *mc;
346
347 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
348 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters.source_vhca_port);
349 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS);
350
351 MLX5_SET_CFG(in, start_flow_index, ix);
352 ix += MLX5_TX_ROCE_GROUP_SIZE;
353 MLX5_SET_CFG(in, end_flow_index, ix - 1);
354 g = mlx5_create_flow_group(roce->ft, in);
355 if (IS_ERR(g)) {
356 err = PTR_ERR(g);
357 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
358 return err;
359 }
360 roce->g = g;
361
362 err = ipsec_fs_roce_tx_mpv_rule_setup(mdev, roce, pol_ft);
363 if (err) {
364 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
365 goto destroy_group;
366 }
367
368 return 0;
369
370 destroy_group:
371 mlx5_destroy_flow_group(roce->g);
372 return err;
373 }
374
ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_table * pol_ft,u32 * in,bool from_event)375 static int ipsec_fs_roce_tx_mpv_create(struct mlx5_core_dev *mdev,
376 struct mlx5_ipsec_fs *ipsec_roce,
377 struct mlx5_flow_table *pol_ft,
378 u32 *in, bool from_event)
379 {
380 struct mlx5_devcom_comp_dev *tmp = NULL;
381 struct mlx5_ipsec_tx_roce *roce;
382 struct mlx5e_priv *peer_priv;
383 int err;
384
385 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
386 return -EOPNOTSUPP;
387
388 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
389 if (!peer_priv) {
390 err = -EOPNOTSUPP;
391 goto release_peer;
392 }
393
394 roce = &ipsec_roce->tx;
395
396 err = ipsec_fs_roce_tx_mpv_create_ft(mdev, roce, pol_ft, peer_priv, from_event);
397 if (err) {
398 mlx5_core_err(mdev, "Fail to create RoCE IPsec tables err=%d\n", err);
399 goto release_peer;
400 }
401
402 err = ipsec_fs_roce_tx_mpv_create_group_rules(mdev, roce, pol_ft, in);
403 if (err) {
404 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group/rule err=%d\n", err);
405 goto destroy_tables;
406 }
407
408 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
409 return 0;
410
411 destroy_tables:
412 mlx5_destroy_flow_table(roce->ft);
413 mlx5_destroy_flow_table(roce->goto_alias_ft);
414 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, roce->alias_id,
415 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
416 release_peer:
417 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
418 return err;
419 }
420
roce_rx_mpv_destroy_tables(struct mlx5_core_dev * mdev,struct mlx5_ipsec_rx_roce * roce)421 static void roce_rx_mpv_destroy_tables(struct mlx5_core_dev *mdev, struct mlx5_ipsec_rx_roce *roce)
422 {
423 mlx5_destroy_flow_table(roce->goto_alias_ft);
424 mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
425 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
426 mlx5_destroy_flow_group(roce->nic_master_group);
427 mlx5_destroy_flow_table(roce->nic_master_ft);
428 }
429
430 #define MLX5_RX_ROCE_GROUP_SIZE BIT(0)
431 #define MLX5_IPSEC_RX_IPV4_FT_LEVEL 3
432 #define MLX5_IPSEC_RX_IPV6_FT_LEVEL 2
433
ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_namespace * ns,u32 family,u32 level,u32 prio)434 static int ipsec_fs_roce_rx_mpv_create(struct mlx5_core_dev *mdev,
435 struct mlx5_ipsec_fs *ipsec_roce,
436 struct mlx5_flow_namespace *ns,
437 u32 family, u32 level, u32 prio)
438 {
439 struct mlx5_flow_namespace *roce_ns, *nic_ns;
440 struct mlx5_flow_table_attr ft_attr = {};
441 struct mlx5_devcom_comp_dev *tmp = NULL;
442 struct mlx5_ipsec_rx_roce *roce;
443 struct mlx5_flow_table next_ft;
444 struct mlx5_flow_table *ft;
445 struct mlx5_flow_group *g;
446 struct mlx5e_priv *peer_priv;
447 int ix = 0;
448 u32 *in;
449 int err;
450
451 roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
452 &ipsec_roce->ipv6_rx;
453
454 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
455 return -EOPNOTSUPP;
456
457 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
458 if (!peer_priv) {
459 err = -EOPNOTSUPP;
460 goto release_peer;
461 }
462
463 roce_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
464 if (!roce_ns) {
465 err = -EOPNOTSUPP;
466 goto release_peer;
467 }
468
469 nic_ns = mlx5_get_flow_namespace(peer_priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
470 if (!nic_ns) {
471 err = -EOPNOTSUPP;
472 goto release_peer;
473 }
474
475 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
476 if (!in) {
477 err = -ENOMEM;
478 goto release_peer;
479 }
480
481 ft_attr.level = (family == AF_INET) ? MLX5_IPSEC_RX_IPV4_FT_LEVEL :
482 MLX5_IPSEC_RX_IPV6_FT_LEVEL;
483 ft_attr.max_fte = 1;
484 ft = mlx5_create_flow_table(roce_ns, &ft_attr);
485 if (IS_ERR(ft)) {
486 err = PTR_ERR(ft);
487 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma master err=%d\n", err);
488 goto free_in;
489 }
490
491 roce->ft_rdma = ft;
492
493 ft_attr.max_fte = 1;
494 ft_attr.prio = prio;
495 ft_attr.level = level + 2;
496 ft = mlx5_create_flow_table(nic_ns, &ft_attr);
497 if (IS_ERR(ft)) {
498 err = PTR_ERR(ft);
499 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC master err=%d\n", err);
500 goto destroy_ft_rdma;
501 }
502 roce->nic_master_ft = ft;
503
504 MLX5_SET_CFG(in, start_flow_index, ix);
505 ix += 1;
506 MLX5_SET_CFG(in, end_flow_index, ix - 1);
507 g = mlx5_create_flow_group(roce->nic_master_ft, in);
508 if (IS_ERR(g)) {
509 err = PTR_ERR(g);
510 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group aliased err=%d\n", err);
511 goto destroy_nic_master_ft;
512 }
513 roce->nic_master_group = g;
514
515 err = ipsec_fs_create_aliased_ft(peer_priv->mdev, mdev, roce->nic_master_ft,
516 &roce->alias_id, roce->key, false);
517 if (err) {
518 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias FT err=%d\n", err);
519 goto destroy_group;
520 }
521
522 next_ft.id = roce->alias_id;
523 ft_attr.max_fte = 1;
524 ft_attr.prio = prio;
525 ft_attr.level = roce->ft->level + 1;
526 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
527 ft_attr.next_ft = &next_ft;
528 ft = mlx5_create_flow_table(ns, &ft_attr);
529 if (IS_ERR(ft)) {
530 err = PTR_ERR(ft);
531 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at NIC slave err=%d\n", err);
532 goto destroy_alias;
533 }
534 roce->goto_alias_ft = ft;
535
536 kvfree(in);
537 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
538 return 0;
539
540 destroy_alias:
541 mlx5_cmd_alias_obj_destroy(mdev, roce->alias_id,
542 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
543 destroy_group:
544 mlx5_destroy_flow_group(roce->nic_master_group);
545 destroy_nic_master_ft:
546 mlx5_destroy_flow_table(roce->nic_master_ft);
547 destroy_ft_rdma:
548 mlx5_destroy_flow_table(roce->ft_rdma);
549 free_in:
550 kvfree(in);
551 release_peer:
552 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
553 return err;
554 }
555
mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_core_dev * mdev)556 void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce,
557 struct mlx5_core_dev *mdev)
558 {
559 struct mlx5_devcom_comp_dev *tmp = NULL;
560 struct mlx5_ipsec_tx_roce *tx_roce;
561 struct mlx5e_priv *peer_priv;
562
563 if (!ipsec_roce)
564 return;
565
566 tx_roce = &ipsec_roce->tx;
567
568 if (!tx_roce->ft)
569 return; /* Incase RoCE was cleaned from MPV event flow */
570
571 mlx5_del_flow_rules(tx_roce->rule);
572 mlx5_destroy_flow_group(tx_roce->g);
573 mlx5_destroy_flow_table(tx_roce->ft);
574
575 if (!mlx5_core_is_mp_slave(mdev))
576 return;
577
578 if (!mlx5_devcom_for_each_peer_begin(*ipsec_roce->devcom))
579 return;
580
581 peer_priv = mlx5_devcom_get_next_peer_data(*ipsec_roce->devcom, &tmp);
582 if (!peer_priv) {
583 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
584 return;
585 }
586
587 mlx5_destroy_flow_table(tx_roce->goto_alias_ft);
588 mlx5_cmd_alias_obj_destroy(peer_priv->mdev, tx_roce->alias_id,
589 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
590 mlx5_devcom_for_each_peer_end(*ipsec_roce->devcom);
591 tx_roce->ft = NULL;
592 }
593
mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_table * pol_ft,bool from_event)594 int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev,
595 struct mlx5_ipsec_fs *ipsec_roce,
596 struct mlx5_flow_table *pol_ft,
597 bool from_event)
598 {
599 struct mlx5_flow_table_attr ft_attr = {};
600 struct mlx5_ipsec_tx_roce *roce;
601 struct mlx5_flow_table *ft;
602 struct mlx5_flow_group *g;
603 int ix = 0;
604 int err;
605 u32 *in;
606
607 if (!ipsec_roce)
608 return 0;
609
610 roce = &ipsec_roce->tx;
611
612 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
613 if (!in)
614 return -ENOMEM;
615
616 if (mlx5_core_is_mp_slave(mdev)) {
617 err = ipsec_fs_roce_tx_mpv_create(mdev, ipsec_roce, pol_ft, in, from_event);
618 goto free_in;
619 }
620
621 ft_attr.max_fte = 1;
622 ft_attr.prio = 1;
623 ft_attr.level = MLX5_IPSEC_RDMA_TX_FT_LEVEL;
624 ft = mlx5_create_flow_table(roce->ns, &ft_attr);
625 if (IS_ERR(ft)) {
626 err = PTR_ERR(ft);
627 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
628 goto free_in;
629 }
630
631 roce->ft = ft;
632
633 MLX5_SET_CFG(in, start_flow_index, ix);
634 ix += MLX5_TX_ROCE_GROUP_SIZE;
635 MLX5_SET_CFG(in, end_flow_index, ix - 1);
636 g = mlx5_create_flow_group(ft, in);
637 if (IS_ERR(g)) {
638 err = PTR_ERR(g);
639 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
640 goto destroy_table;
641 }
642 roce->g = g;
643
644 err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft);
645 if (err) {
646 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
647 goto destroy_group;
648 }
649
650 kvfree(in);
651 return 0;
652
653 destroy_group:
654 mlx5_destroy_flow_group(roce->g);
655 destroy_table:
656 mlx5_destroy_flow_table(ft);
657 free_in:
658 kvfree(in);
659 return err;
660 }
661
mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs * ipsec_roce,u32 family)662 struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family)
663 {
664 struct mlx5_ipsec_rx_roce *rx_roce;
665
666 if (!ipsec_roce)
667 return NULL;
668
669 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
670 &ipsec_roce->ipv6_rx;
671
672 return rx_roce->ft;
673 }
674
mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs * ipsec_roce,u32 family,struct mlx5_core_dev * mdev)675 void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family,
676 struct mlx5_core_dev *mdev)
677 {
678 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
679 struct mlx5_ipsec_rx_roce *rx_roce;
680
681 if (!ipsec_roce)
682 return;
683
684 rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
685 &ipsec_roce->ipv6_rx;
686 if (!rx_roce->ft)
687 return; /* Incase RoCE was cleaned from MPV event flow */
688
689 if (is_mpv_slave)
690 mlx5_del_flow_rules(rx_roce->nic_master_rule);
691 mlx5_del_flow_rules(rx_roce->roce_miss.rule);
692 mlx5_del_flow_rules(rx_roce->rule);
693 if (is_mpv_slave)
694 roce_rx_mpv_destroy_tables(mdev, rx_roce);
695 mlx5_destroy_flow_table(rx_roce->ft_rdma);
696 mlx5_destroy_flow_group(rx_roce->roce_miss.group);
697 mlx5_destroy_flow_group(rx_roce->g);
698 mlx5_destroy_flow_table(rx_roce->ft);
699 rx_roce->ft = NULL;
700 }
701
mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_namespace * ns,struct mlx5_flow_destination * default_dst,u32 family,u32 level,u32 prio)702 int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev,
703 struct mlx5_ipsec_fs *ipsec_roce,
704 struct mlx5_flow_namespace *ns,
705 struct mlx5_flow_destination *default_dst,
706 u32 family, u32 level, u32 prio)
707 {
708 bool is_mpv_slave = mlx5_core_is_mp_slave(mdev);
709 struct mlx5_flow_table_attr ft_attr = {};
710 struct mlx5_ipsec_rx_roce *roce;
711 struct mlx5_flow_table *ft;
712 struct mlx5_flow_group *g;
713 void *outer_headers_c;
714 int ix = 0;
715 u32 *in;
716 int err;
717 u8 *mc;
718
719 if (!ipsec_roce)
720 return 0;
721
722 roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
723 &ipsec_roce->ipv6_rx;
724
725 ft_attr.max_fte = 2;
726 ft_attr.level = level;
727 ft_attr.prio = prio;
728 ft = mlx5_create_flow_table(ns, &ft_attr);
729 if (IS_ERR(ft)) {
730 err = PTR_ERR(ft);
731 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err);
732 return err;
733 }
734
735 roce->ft = ft;
736
737 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
738 if (!in) {
739 err = -ENOMEM;
740 goto fail_nomem;
741 }
742
743 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
744 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
745 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
746 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
747
748 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
749 MLX5_SET_CFG(in, start_flow_index, ix);
750 ix += MLX5_RX_ROCE_GROUP_SIZE;
751 MLX5_SET_CFG(in, end_flow_index, ix - 1);
752 g = mlx5_create_flow_group(ft, in);
753 if (IS_ERR(g)) {
754 err = PTR_ERR(g);
755 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err);
756 goto fail_group;
757 }
758 roce->g = g;
759
760 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
761 MLX5_SET_CFG(in, start_flow_index, ix);
762 ix += MLX5_RX_ROCE_GROUP_SIZE;
763 MLX5_SET_CFG(in, end_flow_index, ix - 1);
764 g = mlx5_create_flow_group(ft, in);
765 if (IS_ERR(g)) {
766 err = PTR_ERR(g);
767 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err);
768 goto fail_mgroup;
769 }
770 roce->roce_miss.group = g;
771
772 if (is_mpv_slave) {
773 err = ipsec_fs_roce_rx_mpv_create(mdev, ipsec_roce, ns, family, level, prio);
774 if (err) {
775 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx alias err=%d\n", err);
776 goto fail_mpv_create;
777 }
778 } else {
779 memset(&ft_attr, 0, sizeof(ft_attr));
780 if (family == AF_INET)
781 ft_attr.level = 1;
782 ft_attr.max_fte = 1;
783 ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr);
784 if (IS_ERR(ft)) {
785 err = PTR_ERR(ft);
786 mlx5_core_err(mdev,
787 "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err);
788 goto fail_rdma_table;
789 }
790
791 roce->ft_rdma = ft;
792 }
793
794 err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce);
795 if (err) {
796 mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err);
797 goto fail_setup_rule;
798 }
799
800 kvfree(in);
801 return 0;
802
803 fail_setup_rule:
804 if (is_mpv_slave)
805 roce_rx_mpv_destroy_tables(mdev, roce);
806 mlx5_destroy_flow_table(roce->ft_rdma);
807 fail_mpv_create:
808 fail_rdma_table:
809 mlx5_destroy_flow_group(roce->roce_miss.group);
810 fail_mgroup:
811 mlx5_destroy_flow_group(roce->g);
812 fail_group:
813 kvfree(in);
814 fail_nomem:
815 mlx5_destroy_flow_table(roce->ft);
816 return err;
817 }
818
mlx5_ipsec_fs_is_mpv_roce_supported(struct mlx5_core_dev * mdev)819 bool mlx5_ipsec_fs_is_mpv_roce_supported(struct mlx5_core_dev *mdev)
820 {
821 if (!mlx5_core_mp_enabled(mdev))
822 return true;
823
824 if (ipsec_fs_create_alias_supported_one(mdev))
825 return true;
826
827 return false;
828 }
829
mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs * ipsec_roce)830 void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce)
831 {
832 kfree(ipsec_roce);
833 }
834
mlx5_ipsec_fs_roce_init(struct mlx5_core_dev * mdev,struct mlx5_devcom_comp_dev ** devcom)835 struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev,
836 struct mlx5_devcom_comp_dev **devcom)
837 {
838 struct mlx5_ipsec_fs *roce_ipsec;
839 struct mlx5_flow_namespace *ns;
840
841 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
842 if (!ns) {
843 mlx5_core_err(mdev, "Failed to get RoCE rx ns\n");
844 return NULL;
845 }
846
847 roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL);
848 if (!roce_ipsec)
849 return NULL;
850
851 roce_ipsec->ipv4_rx.ns_rdma = ns;
852 roce_ipsec->ipv6_rx.ns_rdma = ns;
853
854 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
855 if (!ns) {
856 mlx5_core_err(mdev, "Failed to get RoCE tx ns\n");
857 goto err_tx;
858 }
859
860 roce_ipsec->tx.ns = ns;
861
862 roce_ipsec->devcom = devcom;
863
864 return roce_ipsec;
865
866 err_tx:
867 kfree(roce_ipsec);
868 return NULL;
869 }
870