1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/uverbs_std_types.h>
11 #include <rdma/mlx5_user_ioctl_cmds.h>
12 #include <rdma/mlx5_user_ioctl_verbs.h>
13 #include <rdma/ib_hdrs.h>
14 #include <rdma/ib_umem.h>
15 #include <rdma/ib_ucaps.h>
16 #include <linux/mlx5/driver.h>
17 #include <linux/mlx5/fs.h>
18 #include <linux/mlx5/fs_helpers.h>
19 #include <linux/mlx5/eswitch.h>
20 #include <net/inet_ecn.h>
21 #include "mlx5_ib.h"
22 #include "counters.h"
23 #include "devx.h"
24 #include "fs.h"
25
26 #define UVERBS_MODULE_NAME mlx5_ib
27 #include <rdma/uverbs_named_ioctl.h>
28
29 enum {
30 MATCH_CRITERIA_ENABLE_OUTER_BIT,
31 MATCH_CRITERIA_ENABLE_MISC_BIT,
32 MATCH_CRITERIA_ENABLE_INNER_BIT,
33 MATCH_CRITERIA_ENABLE_MISC2_BIT
34 };
35
36
37 struct mlx5_per_qp_opfc {
38 struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];
39 };
40
41 #define HEADER_IS_ZERO(match_criteria, headers) \
42 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
43 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
44
get_match_criteria_enable(u32 * match_criteria)45 static u8 get_match_criteria_enable(u32 *match_criteria)
46 {
47 u8 match_criteria_enable;
48
49 match_criteria_enable =
50 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
51 MATCH_CRITERIA_ENABLE_OUTER_BIT;
52 match_criteria_enable |=
53 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
54 MATCH_CRITERIA_ENABLE_MISC_BIT;
55 match_criteria_enable |=
56 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
57 MATCH_CRITERIA_ENABLE_INNER_BIT;
58 match_criteria_enable |=
59 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
60 MATCH_CRITERIA_ENABLE_MISC2_BIT;
61
62 return match_criteria_enable;
63 }
64
set_proto(void * outer_c,void * outer_v,u8 mask,u8 val)65 static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
66 {
67 u8 entry_mask;
68 u8 entry_val;
69 int err = 0;
70
71 if (!mask)
72 goto out;
73
74 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
75 ip_protocol);
76 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
77 ip_protocol);
78 if (!entry_mask) {
79 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
80 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
81 goto out;
82 }
83 /* Don't override existing ip protocol */
84 if (mask != entry_mask || val != entry_val)
85 err = -EINVAL;
86 out:
87 return err;
88 }
89
set_flow_label(void * misc_c,void * misc_v,u32 mask,u32 val,bool inner)90 static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
91 bool inner)
92 {
93 if (inner) {
94 MLX5_SET(fte_match_set_misc,
95 misc_c, inner_ipv6_flow_label, mask);
96 MLX5_SET(fte_match_set_misc,
97 misc_v, inner_ipv6_flow_label, val);
98 } else {
99 MLX5_SET(fte_match_set_misc,
100 misc_c, outer_ipv6_flow_label, mask);
101 MLX5_SET(fte_match_set_misc,
102 misc_v, outer_ipv6_flow_label, val);
103 }
104 }
105
set_tos(void * outer_c,void * outer_v,u8 mask,u8 val)106 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
107 {
108 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
109 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
110 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
111 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
112 }
113
check_mpls_supp_fields(u32 field_support,const __be32 * set_mask)114 static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
115 {
116 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
117 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
118 return -EOPNOTSUPP;
119
120 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
121 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
122 return -EOPNOTSUPP;
123
124 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
125 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
126 return -EOPNOTSUPP;
127
128 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
129 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
130 return -EOPNOTSUPP;
131
132 return 0;
133 }
134
135 #define LAST_ETH_FIELD vlan_tag
136 #define LAST_IPV4_FIELD tos
137 #define LAST_IPV6_FIELD traffic_class
138 #define LAST_TCP_UDP_FIELD src_port
139 #define LAST_TUNNEL_FIELD tunnel_id
140 #define LAST_FLOW_TAG_FIELD tag_id
141 #define LAST_DROP_FIELD size
142 #define LAST_COUNTERS_FIELD counters
143
144 /* Field is the last supported field */
145 #define FIELDS_NOT_SUPPORTED(filter, field) \
146 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
147 sizeof(filter) - offsetofend(typeof(filter), field))
148
parse_flow_flow_action(struct mlx5_ib_flow_action * maction,bool is_egress,struct mlx5_flow_act * action)149 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
150 bool is_egress,
151 struct mlx5_flow_act *action)
152 {
153
154 switch (maction->ib_action.type) {
155 case IB_FLOW_ACTION_UNSPECIFIED:
156 if (maction->flow_action_raw.sub_type ==
157 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
158 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
159 return -EINVAL;
160 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
161 action->modify_hdr =
162 maction->flow_action_raw.modify_hdr;
163 return 0;
164 }
165 if (maction->flow_action_raw.sub_type ==
166 MLX5_IB_FLOW_ACTION_DECAP) {
167 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
168 return -EINVAL;
169 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
170 return 0;
171 }
172 if (maction->flow_action_raw.sub_type ==
173 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
174 if (action->action &
175 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
176 return -EINVAL;
177 action->action |=
178 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
179 action->pkt_reformat =
180 maction->flow_action_raw.pkt_reformat;
181 return 0;
182 }
183 fallthrough;
184 default:
185 return -EOPNOTSUPP;
186 }
187 }
188
parse_flow_attr(struct mlx5_core_dev * mdev,struct mlx5_flow_spec * spec,const union ib_flow_spec * ib_spec,const struct ib_flow_attr * flow_attr,struct mlx5_flow_act * action,u32 prev_type)189 static int parse_flow_attr(struct mlx5_core_dev *mdev,
190 struct mlx5_flow_spec *spec,
191 const union ib_flow_spec *ib_spec,
192 const struct ib_flow_attr *flow_attr,
193 struct mlx5_flow_act *action, u32 prev_type)
194 {
195 struct mlx5_flow_context *flow_context = &spec->flow_context;
196 u32 *match_c = spec->match_criteria;
197 u32 *match_v = spec->match_value;
198 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
199 misc_parameters);
200 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
201 misc_parameters);
202 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
203 misc_parameters_2);
204 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
205 misc_parameters_2);
206 void *headers_c;
207 void *headers_v;
208 int match_ipv;
209 int ret;
210
211 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
212 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
213 inner_headers);
214 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
215 inner_headers);
216 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
217 ft_field_support.inner_ip_version);
218 } else {
219 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
220 outer_headers);
221 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
222 outer_headers);
223 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
224 ft_field_support.outer_ip_version);
225 }
226
227 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
228 case IB_FLOW_SPEC_ETH:
229 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
230 return -EOPNOTSUPP;
231
232 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
233 dmac_47_16),
234 ib_spec->eth.mask.dst_mac);
235 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
236 dmac_47_16),
237 ib_spec->eth.val.dst_mac);
238
239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
240 smac_47_16),
241 ib_spec->eth.mask.src_mac);
242 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
243 smac_47_16),
244 ib_spec->eth.val.src_mac);
245
246 if (ib_spec->eth.mask.vlan_tag) {
247 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
248 cvlan_tag, 1);
249 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
250 cvlan_tag, 1);
251
252 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
253 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
254 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
255 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
256
257 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
258 first_cfi,
259 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
260 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
261 first_cfi,
262 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
263
264 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
265 first_prio,
266 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
267 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
268 first_prio,
269 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
270 }
271 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
272 ethertype, ntohs(ib_spec->eth.mask.ether_type));
273 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
274 ethertype, ntohs(ib_spec->eth.val.ether_type));
275 break;
276 case IB_FLOW_SPEC_IPV4:
277 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
278 return -EOPNOTSUPP;
279
280 if (match_ipv) {
281 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
282 ip_version, 0xf);
283 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
284 ip_version, MLX5_FS_IPV4_VERSION);
285 } else {
286 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
287 ethertype, 0xffff);
288 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
289 ethertype, ETH_P_IP);
290 }
291
292 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
293 src_ipv4_src_ipv6.ipv4_layout.ipv4),
294 &ib_spec->ipv4.mask.src_ip,
295 sizeof(ib_spec->ipv4.mask.src_ip));
296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
297 src_ipv4_src_ipv6.ipv4_layout.ipv4),
298 &ib_spec->ipv4.val.src_ip,
299 sizeof(ib_spec->ipv4.val.src_ip));
300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
301 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
302 &ib_spec->ipv4.mask.dst_ip,
303 sizeof(ib_spec->ipv4.mask.dst_ip));
304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
306 &ib_spec->ipv4.val.dst_ip,
307 sizeof(ib_spec->ipv4.val.dst_ip));
308
309 set_tos(headers_c, headers_v,
310 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
311
312 if (set_proto(headers_c, headers_v,
313 ib_spec->ipv4.mask.proto,
314 ib_spec->ipv4.val.proto))
315 return -EINVAL;
316 break;
317 case IB_FLOW_SPEC_IPV6:
318 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
319 return -EOPNOTSUPP;
320
321 if (match_ipv) {
322 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
323 ip_version, 0xf);
324 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
325 ip_version, MLX5_FS_IPV6_VERSION);
326 } else {
327 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
328 ethertype, 0xffff);
329 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
330 ethertype, ETH_P_IPV6);
331 }
332
333 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
334 src_ipv4_src_ipv6.ipv6_layout.ipv6),
335 &ib_spec->ipv6.mask.src_ip,
336 sizeof(ib_spec->ipv6.mask.src_ip));
337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
338 src_ipv4_src_ipv6.ipv6_layout.ipv6),
339 &ib_spec->ipv6.val.src_ip,
340 sizeof(ib_spec->ipv6.val.src_ip));
341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
342 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
343 &ib_spec->ipv6.mask.dst_ip,
344 sizeof(ib_spec->ipv6.mask.dst_ip));
345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
347 &ib_spec->ipv6.val.dst_ip,
348 sizeof(ib_spec->ipv6.val.dst_ip));
349
350 set_tos(headers_c, headers_v,
351 ib_spec->ipv6.mask.traffic_class,
352 ib_spec->ipv6.val.traffic_class);
353
354 if (set_proto(headers_c, headers_v,
355 ib_spec->ipv6.mask.next_hdr,
356 ib_spec->ipv6.val.next_hdr))
357 return -EINVAL;
358
359 set_flow_label(misc_params_c, misc_params_v,
360 ntohl(ib_spec->ipv6.mask.flow_label),
361 ntohl(ib_spec->ipv6.val.flow_label),
362 ib_spec->type & IB_FLOW_SPEC_INNER);
363 break;
364 case IB_FLOW_SPEC_ESP:
365 return -EOPNOTSUPP;
366 case IB_FLOW_SPEC_TCP:
367 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
368 LAST_TCP_UDP_FIELD))
369 return -EOPNOTSUPP;
370
371 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
372 return -EINVAL;
373
374 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
375 ntohs(ib_spec->tcp_udp.mask.src_port));
376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
377 ntohs(ib_spec->tcp_udp.val.src_port));
378
379 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
380 ntohs(ib_spec->tcp_udp.mask.dst_port));
381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
382 ntohs(ib_spec->tcp_udp.val.dst_port));
383 break;
384 case IB_FLOW_SPEC_UDP:
385 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
386 LAST_TCP_UDP_FIELD))
387 return -EOPNOTSUPP;
388
389 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
390 return -EINVAL;
391
392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
393 ntohs(ib_spec->tcp_udp.mask.src_port));
394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
395 ntohs(ib_spec->tcp_udp.val.src_port));
396
397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
398 ntohs(ib_spec->tcp_udp.mask.dst_port));
399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
400 ntohs(ib_spec->tcp_udp.val.dst_port));
401 break;
402 case IB_FLOW_SPEC_GRE:
403 if (ib_spec->gre.mask.c_ks_res0_ver)
404 return -EOPNOTSUPP;
405
406 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
407 return -EINVAL;
408
409 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
410 0xff);
411 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
412 IPPROTO_GRE);
413
414 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
415 ntohs(ib_spec->gre.mask.protocol));
416 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
417 ntohs(ib_spec->gre.val.protocol));
418
419 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
420 gre_key.nvgre.hi),
421 &ib_spec->gre.mask.key,
422 sizeof(ib_spec->gre.mask.key));
423 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
424 gre_key.nvgre.hi),
425 &ib_spec->gre.val.key,
426 sizeof(ib_spec->gre.val.key));
427 break;
428 case IB_FLOW_SPEC_MPLS:
429 switch (prev_type) {
430 case IB_FLOW_SPEC_UDP:
431 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
432 ft_field_support.outer_first_mpls_over_udp),
433 &ib_spec->mpls.mask.tag))
434 return -EOPNOTSUPP;
435
436 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
437 outer_first_mpls_over_udp),
438 &ib_spec->mpls.val.tag,
439 sizeof(ib_spec->mpls.val.tag));
440 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
441 outer_first_mpls_over_udp),
442 &ib_spec->mpls.mask.tag,
443 sizeof(ib_spec->mpls.mask.tag));
444 break;
445 case IB_FLOW_SPEC_GRE:
446 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
447 ft_field_support.outer_first_mpls_over_gre),
448 &ib_spec->mpls.mask.tag))
449 return -EOPNOTSUPP;
450
451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
452 outer_first_mpls_over_gre),
453 &ib_spec->mpls.val.tag,
454 sizeof(ib_spec->mpls.val.tag));
455 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
456 outer_first_mpls_over_gre),
457 &ib_spec->mpls.mask.tag,
458 sizeof(ib_spec->mpls.mask.tag));
459 break;
460 default:
461 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
462 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
463 ft_field_support.inner_first_mpls),
464 &ib_spec->mpls.mask.tag))
465 return -EOPNOTSUPP;
466
467 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
468 inner_first_mpls),
469 &ib_spec->mpls.val.tag,
470 sizeof(ib_spec->mpls.val.tag));
471 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
472 inner_first_mpls),
473 &ib_spec->mpls.mask.tag,
474 sizeof(ib_spec->mpls.mask.tag));
475 } else {
476 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
477 ft_field_support.outer_first_mpls),
478 &ib_spec->mpls.mask.tag))
479 return -EOPNOTSUPP;
480
481 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
482 outer_first_mpls),
483 &ib_spec->mpls.val.tag,
484 sizeof(ib_spec->mpls.val.tag));
485 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
486 outer_first_mpls),
487 &ib_spec->mpls.mask.tag,
488 sizeof(ib_spec->mpls.mask.tag));
489 }
490 }
491 break;
492 case IB_FLOW_SPEC_VXLAN_TUNNEL:
493 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
494 LAST_TUNNEL_FIELD))
495 return -EOPNOTSUPP;
496
497 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
498 ntohl(ib_spec->tunnel.mask.tunnel_id));
499 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
500 ntohl(ib_spec->tunnel.val.tunnel_id));
501 break;
502 case IB_FLOW_SPEC_ACTION_TAG:
503 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
504 LAST_FLOW_TAG_FIELD))
505 return -EOPNOTSUPP;
506 if (ib_spec->flow_tag.tag_id >= BIT(24))
507 return -EINVAL;
508
509 flow_context->flow_tag = ib_spec->flow_tag.tag_id;
510 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
511 break;
512 case IB_FLOW_SPEC_ACTION_DROP:
513 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
514 LAST_DROP_FIELD))
515 return -EOPNOTSUPP;
516 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
517 break;
518 case IB_FLOW_SPEC_ACTION_HANDLE:
519 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
520 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
521 if (ret)
522 return ret;
523 break;
524 case IB_FLOW_SPEC_ACTION_COUNT:
525 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
526 LAST_COUNTERS_FIELD))
527 return -EOPNOTSUPP;
528
529 /* for now support only one counters spec per flow */
530 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
531 return -EINVAL;
532
533 action->counters = ib_spec->flow_count.counters;
534 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
535 break;
536 default:
537 return -EINVAL;
538 }
539
540 return 0;
541 }
542
543 /* If a flow could catch both multicast and unicast packets,
544 * it won't fall into the multicast flow steering table and this rule
545 * could steal other multicast packets.
546 */
flow_is_multicast_only(const struct ib_flow_attr * ib_attr)547 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
548 {
549 union ib_flow_spec *flow_spec;
550
551 if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
552 ib_attr->num_of_specs < 1)
553 return false;
554
555 flow_spec = (union ib_flow_spec *)(ib_attr + 1);
556 if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
557 struct ib_flow_spec_ipv4 *ipv4_spec;
558
559 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
560 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
561 return true;
562
563 return false;
564 }
565
566 if (flow_spec->type == IB_FLOW_SPEC_ETH) {
567 struct ib_flow_spec_eth *eth_spec;
568
569 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
570 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
571 is_multicast_ether_addr(eth_spec->val.dst_mac);
572 }
573
574 return false;
575 }
576
is_valid_ethertype(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr,bool check_inner)577 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
578 const struct ib_flow_attr *flow_attr,
579 bool check_inner)
580 {
581 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
582 int match_ipv = check_inner ?
583 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
584 ft_field_support.inner_ip_version) :
585 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
586 ft_field_support.outer_ip_version);
587 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
588 bool ipv4_spec_valid, ipv6_spec_valid;
589 unsigned int ip_spec_type = 0;
590 bool has_ethertype = false;
591 unsigned int spec_index;
592 bool mask_valid = true;
593 u16 eth_type = 0;
594 bool type_valid;
595
596 /* Validate that ethertype is correct */
597 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
598 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
599 ib_spec->eth.mask.ether_type) {
600 mask_valid = (ib_spec->eth.mask.ether_type ==
601 htons(0xffff));
602 has_ethertype = true;
603 eth_type = ntohs(ib_spec->eth.val.ether_type);
604 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
605 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
606 ip_spec_type = ib_spec->type;
607 }
608 ib_spec = (void *)ib_spec + ib_spec->size;
609 }
610
611 type_valid = (!has_ethertype) || (!ip_spec_type);
612 if (!type_valid && mask_valid) {
613 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
614 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
615 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
616 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
617
618 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
619 (((eth_type == ETH_P_MPLS_UC) ||
620 (eth_type == ETH_P_MPLS_MC)) && match_ipv);
621 }
622
623 return type_valid;
624 }
625
is_valid_attr(struct mlx5_core_dev * mdev,const struct ib_flow_attr * flow_attr)626 static bool is_valid_attr(struct mlx5_core_dev *mdev,
627 const struct ib_flow_attr *flow_attr)
628 {
629 return is_valid_ethertype(mdev, flow_attr, false) &&
630 is_valid_ethertype(mdev, flow_attr, true);
631 }
632
put_flow_table(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * prio,bool ft_added)633 static void put_flow_table(struct mlx5_ib_dev *dev,
634 struct mlx5_ib_flow_prio *prio, bool ft_added)
635 {
636 prio->refcount -= !!ft_added;
637 if (!prio->refcount) {
638 mlx5_destroy_flow_table(prio->flow_table);
639 prio->flow_table = NULL;
640 }
641 }
642
mlx5_ib_destroy_flow(struct ib_flow * flow_id)643 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
644 {
645 struct mlx5_ib_flow_handler *handler = container_of(flow_id,
646 struct mlx5_ib_flow_handler,
647 ibflow);
648 struct mlx5_ib_flow_handler *iter, *tmp;
649 struct mlx5_ib_dev *dev = handler->dev;
650
651 mutex_lock(&dev->flow_db->lock);
652
653 list_for_each_entry_safe(iter, tmp, &handler->list, list) {
654 mlx5_del_flow_rules(iter->rule);
655 put_flow_table(dev, iter->prio, true);
656 list_del(&iter->list);
657 kfree(iter);
658 }
659
660 mlx5_del_flow_rules(handler->rule);
661 put_flow_table(dev, handler->prio, true);
662 mlx5_ib_counters_clear_description(handler->ibcounters);
663 mutex_unlock(&dev->flow_db->lock);
664 if (handler->flow_matcher)
665 atomic_dec(&handler->flow_matcher->usecnt);
666 kfree(handler);
667
668 return 0;
669 }
670
ib_prio_to_core_prio(unsigned int priority,bool dont_trap)671 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
672 {
673 priority *= 2;
674 if (!dont_trap)
675 priority++;
676 return priority;
677 }
678
679 enum flow_table_type {
680 MLX5_IB_FT_RX,
681 MLX5_IB_FT_TX
682 };
683
684 #define MLX5_FS_MAX_TYPES 6
685 #define MLX5_FS_MAX_ENTRIES BIT(16)
686
mlx5_ib_shared_ft_allowed(struct ib_device * device)687 static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)
688 {
689 struct mlx5_ib_dev *dev = to_mdev(device);
690
691 return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed);
692 }
693
_get_prio(struct mlx5_ib_dev * dev,struct mlx5_flow_namespace * ns,struct mlx5_ib_flow_prio * prio,int priority,int num_entries,int num_groups,u32 flags,u16 vport)694 static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev,
695 struct mlx5_flow_namespace *ns,
696 struct mlx5_ib_flow_prio *prio,
697 int priority,
698 int num_entries, int num_groups,
699 u32 flags, u16 vport)
700 {
701 struct mlx5_flow_table_attr ft_attr = {};
702 struct mlx5_flow_table *ft;
703
704 ft_attr.prio = priority;
705 ft_attr.max_fte = num_entries;
706 ft_attr.flags = flags;
707 ft_attr.vport = vport;
708 ft_attr.autogroup.max_num_groups = num_groups;
709 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
710 if (IS_ERR(ft))
711 return ERR_CAST(ft);
712
713 prio->flow_table = ft;
714 prio->refcount = 0;
715 return prio;
716 }
717
get_flow_table(struct mlx5_ib_dev * dev,struct ib_flow_attr * flow_attr,enum flow_table_type ft_type)718 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
719 struct ib_flow_attr *flow_attr,
720 enum flow_table_type ft_type)
721 {
722 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
723 struct mlx5_flow_namespace *ns = NULL;
724 enum mlx5_flow_namespace_type fn_type;
725 struct mlx5_ib_flow_prio *prio;
726 struct mlx5_flow_table *ft;
727 int max_table_size;
728 int num_entries;
729 int num_groups;
730 bool esw_encap;
731 u32 flags = 0;
732 int priority;
733
734 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
735 log_max_ft_size));
736 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
737 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
738 switch (flow_attr->type) {
739 case IB_FLOW_ATTR_NORMAL:
740 if (flow_is_multicast_only(flow_attr) && !dont_trap)
741 priority = MLX5_IB_FLOW_MCAST_PRIO;
742 else
743 priority = ib_prio_to_core_prio(flow_attr->priority,
744 dont_trap);
745 if (ft_type == MLX5_IB_FT_RX) {
746 fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
747 prio = &dev->flow_db->prios[priority];
748 if (!dev->is_rep && !esw_encap &&
749 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
750 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
751 if (!dev->is_rep && !esw_encap &&
752 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
753 reformat_l3_tunnel_to_l2))
754 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
755 } else {
756 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
757 dev->mdev, log_max_ft_size));
758 fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
759 prio = &dev->flow_db->egress_prios[priority];
760 if (!dev->is_rep && !esw_encap &&
761 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
762 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
763 }
764 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
765 num_entries = MLX5_FS_MAX_ENTRIES;
766 num_groups = MLX5_FS_MAX_TYPES;
767 break;
768 case IB_FLOW_ATTR_ALL_DEFAULT:
769 case IB_FLOW_ATTR_MC_DEFAULT:
770 ns = mlx5_get_flow_namespace(dev->mdev,
771 MLX5_FLOW_NAMESPACE_LEFTOVERS);
772 build_leftovers_ft_param(&priority, &num_entries, &num_groups);
773 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
774 break;
775 case IB_FLOW_ATTR_SNIFFER:
776 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
777 allow_sniffer_and_nic_rx_shared_tir))
778 return ERR_PTR(-EOPNOTSUPP);
779
780 ns = mlx5_get_flow_namespace(
781 dev->mdev, ft_type == MLX5_IB_FT_RX ?
782 MLX5_FLOW_NAMESPACE_SNIFFER_RX :
783 MLX5_FLOW_NAMESPACE_SNIFFER_TX);
784
785 prio = &dev->flow_db->sniffer[ft_type];
786 priority = 0;
787 num_entries = 1;
788 num_groups = 1;
789 break;
790 default:
791 break;
792 }
793
794 if (!ns)
795 return ERR_PTR(-EOPNOTSUPP);
796
797 max_table_size = min_t(int, num_entries, max_table_size);
798
799 ft = prio->flow_table;
800 if (!ft)
801 return _get_prio(dev, ns, prio, priority, max_table_size,
802 num_groups, flags, 0);
803
804 return prio;
805 }
806
807 enum {
808 RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,
809 RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,
810 RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
811 RDMA_RX_ECN_OPCOUNTER_PRIO,
812 RDMA_RX_CNP_OPCOUNTER_PRIO,
813 RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,
814 };
815
816 enum {
817 RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,
818 RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
819 RDMA_TX_CNP_OPCOUNTER_PRIO,
820 RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,
821 };
822
set_vhca_port_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)823 static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
824 struct mlx5_flow_spec *spec)
825 {
826 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
827 ft_field_support.source_vhca_port) ||
828 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
829 ft_field_support.source_vhca_port))
830 return -EOPNOTSUPP;
831
832 MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria,
833 misc_parameters.source_vhca_port);
834 MLX5_SET(fte_match_param, &spec->match_value,
835 misc_parameters.source_vhca_port, port_num);
836
837 return 0;
838 }
839
set_ecn_ce_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec,int ipv)840 static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
841 struct mlx5_flow_spec *spec, int ipv)
842 {
843 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
844 ft_field_support.outer_ip_version))
845 return -EOPNOTSUPP;
846
847 if (mlx5_core_mp_enabled(dev->mdev) &&
848 set_vhca_port_spec(dev, port_num, spec))
849 return -EOPNOTSUPP;
850
851 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
852 outer_headers.ip_ecn);
853 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn,
854 INET_ECN_CE);
855 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
856 outer_headers.ip_version);
857 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
858 ipv);
859
860 spec->match_criteria_enable =
861 get_match_criteria_enable(spec->match_criteria);
862
863 return 0;
864 }
865
set_cnp_spec(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_flow_spec * spec)866 static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
867 struct mlx5_flow_spec *spec)
868 {
869 if (mlx5_core_mp_enabled(dev->mdev) &&
870 set_vhca_port_spec(dev, port_num, spec))
871 return -EOPNOTSUPP;
872
873 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
874 misc_parameters.bth_opcode);
875 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode,
876 IB_BTH_OPCODE_CNP);
877
878 spec->match_criteria_enable =
879 get_match_criteria_enable(spec->match_criteria);
880
881 return 0;
882 }
883
884 /* Returns the prio we should use for the given optional counter type,
885 * whereas for bytes type we use the packet type, since they share the same
886 * resources.
887 */
get_opfc_prio(struct mlx5_ib_dev * dev,u32 type)888 static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,
889 u32 type)
890 {
891 u32 prio_type;
892
893 switch (type) {
894 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
895 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
896 break;
897 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
898 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
899 break;
900 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
901 prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
902 break;
903 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
904 prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
905 break;
906 default:
907 prio_type = type;
908 }
909
910 return &dev->flow_db->opfcs[prio_type];
911 }
912
put_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)913 static void put_per_qp_prio(struct mlx5_ib_dev *dev,
914 enum mlx5_ib_optional_counter_type type)
915 {
916 enum mlx5_ib_optional_counter_type per_qp_type;
917 struct mlx5_ib_flow_prio *prio;
918
919 switch (type) {
920 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
921 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
922 break;
923 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
924 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
925 break;
926 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
927 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
928 break;
929 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
930 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
931 break;
932 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
933 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
934 break;
935 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
936 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
937 break;
938 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
939 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
940 break;
941 default:
942 return;
943 }
944
945 prio = get_opfc_prio(dev, per_qp_type);
946 put_flow_table(dev, prio, true);
947 }
948
get_per_qp_prio(struct mlx5_ib_dev * dev,enum mlx5_ib_optional_counter_type type)949 static int get_per_qp_prio(struct mlx5_ib_dev *dev,
950 enum mlx5_ib_optional_counter_type type)
951 {
952 enum mlx5_ib_optional_counter_type per_qp_type;
953 enum mlx5_flow_namespace_type fn_type;
954 struct mlx5_flow_namespace *ns;
955 struct mlx5_ib_flow_prio *prio;
956 int priority;
957
958 switch (type) {
959 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
960 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
961 priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;
962 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
963 break;
964 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
965 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
966 priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;
967 per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
968 break;
969 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
970 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
971 priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;
972 per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
973 break;
974 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
975 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
976 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
977 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
978 break;
979 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
980 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
981 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
982 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
983 break;
984 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
985 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
986 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
987 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
988 break;
989 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
990 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
991 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
992 per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
993 break;
994 default:
995 return -EINVAL;
996 }
997
998 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
999 if (!ns)
1000 return -EOPNOTSUPP;
1001
1002 prio = get_opfc_prio(dev, per_qp_type);
1003 if (prio->flow_table)
1004 return 0;
1005
1006 prio = _get_prio(dev, ns, prio, priority, MLX5_FS_MAX_POOL_SIZE, 1, 0, 0);
1007 if (IS_ERR(prio))
1008 return PTR_ERR(prio);
1009
1010 prio->refcount = 1;
1011
1012 return 0;
1013 }
1014
get_per_qp_opfc(struct xarray * qpn_opfc_xa,u32 qp_num,bool * new)1015 static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,
1016 u32 qp_num, bool *new)
1017 {
1018 struct mlx5_per_qp_opfc *per_qp_opfc;
1019
1020 *new = false;
1021
1022 per_qp_opfc = xa_load(qpn_opfc_xa, qp_num);
1023 if (per_qp_opfc)
1024 return per_qp_opfc;
1025 per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
1026
1027 if (!per_qp_opfc)
1028 return NULL;
1029
1030 *new = true;
1031 return per_qp_opfc;
1032 }
1033
add_op_fc_rules(struct mlx5_ib_dev * dev,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],struct xarray * qpn_opfc_xa,struct mlx5_per_qp_opfc * per_qp_opfc,struct mlx5_ib_flow_prio * prio,enum mlx5_ib_optional_counter_type type,u32 qp_num,u32 port_num)1034 static int add_op_fc_rules(struct mlx5_ib_dev *dev,
1035 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1036 struct xarray *qpn_opfc_xa,
1037 struct mlx5_per_qp_opfc *per_qp_opfc,
1038 struct mlx5_ib_flow_prio *prio,
1039 enum mlx5_ib_optional_counter_type type,
1040 u32 qp_num, u32 port_num)
1041 {
1042 struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc;
1043 struct mlx5_flow_act flow_act = {};
1044 struct mlx5_flow_destination dst;
1045 struct mlx5_flow_spec *spec;
1046 int i, err, spec_num;
1047 bool is_tx;
1048
1049 if (opfc->fc)
1050 return -EEXIST;
1051
1052 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type,
1053 &in_use_opfc)) {
1054 opfc->fc = in_use_opfc->fc;
1055 opfc->rule[0] = in_use_opfc->rule[0];
1056 return 0;
1057 }
1058
1059 opfc->fc = fc_arr[type];
1060
1061 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1062 if (!spec) {
1063 err = -ENOMEM;
1064 goto null_fc;
1065 }
1066
1067 switch (type) {
1068 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:
1069 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1070 MLX5_FS_IPV4_VERSION) ||
1071 set_ecn_ce_spec(dev, port_num, &spec[1],
1072 MLX5_FS_IPV6_VERSION)) {
1073 err = -EOPNOTSUPP;
1074 goto free_spec;
1075 }
1076 spec_num = 2;
1077 is_tx = false;
1078
1079 MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,
1080 misc_parameters.bth_dst_qp);
1081 MLX5_SET(fte_match_param, spec[1].match_value,
1082 misc_parameters.bth_dst_qp, qp_num);
1083 spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1084 break;
1085 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:
1086 if (!MLX5_CAP_FLOWTABLE(
1087 dev->mdev,
1088 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1089 set_cnp_spec(dev, port_num, &spec[0])) {
1090 err = -EOPNOTSUPP;
1091 goto free_spec;
1092 }
1093 spec_num = 1;
1094 is_tx = false;
1095 break;
1096 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:
1097 if (!MLX5_CAP_FLOWTABLE(
1098 dev->mdev,
1099 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1100 set_cnp_spec(dev, port_num, &spec[0])) {
1101 err = -EOPNOTSUPP;
1102 goto free_spec;
1103 }
1104 spec_num = 1;
1105 is_tx = true;
1106 break;
1107 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1108 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1109 spec_num = 1;
1110 is_tx = true;
1111 break;
1112 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1113 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1114 spec_num = 1;
1115 is_tx = false;
1116 break;
1117 default:
1118 err = -EINVAL;
1119 goto free_spec;
1120 }
1121
1122 if (is_tx) {
1123 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1124 misc_parameters.source_sqn);
1125 MLX5_SET(fte_match_param, spec->match_value,
1126 misc_parameters.source_sqn, qp_num);
1127 } else {
1128 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1129 misc_parameters.bth_dst_qp);
1130 MLX5_SET(fte_match_param, spec->match_value,
1131 misc_parameters.bth_dst_qp, qp_num);
1132 }
1133
1134 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1135
1136 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1137 dst.counter = opfc->fc;
1138
1139 flow_act.action =
1140 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1141
1142 for (i = 0; i < spec_num; i++) {
1143 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1144 &flow_act, &dst, 1);
1145 if (IS_ERR(opfc->rule[i])) {
1146 err = PTR_ERR(opfc->rule[i]);
1147 goto del_rules;
1148 }
1149 }
1150 prio->refcount += spec_num;
1151
1152 err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL));
1153 if (err)
1154 goto del_rules;
1155
1156 kfree(spec);
1157
1158 return 0;
1159
1160 del_rules:
1161 while (i--)
1162 mlx5_del_flow_rules(opfc->rule[i]);
1163 put_flow_table(dev, prio, false);
1164 free_spec:
1165 kfree(spec);
1166 null_fc:
1167 opfc->fc = NULL;
1168 return err;
1169 }
1170
1171 static bool
is_fc_shared_and_in_use(struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],u32 type,struct mlx5_fc ** fc)1172 is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,
1173 struct mlx5_fc **fc)
1174 {
1175 u32 shared_fc_type;
1176
1177 switch (type) {
1178 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1179 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
1180 break;
1181 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1182 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
1183 break;
1184 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1185 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
1186 break;
1187 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1188 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
1189 break;
1190 default:
1191 return false;
1192 }
1193
1194 *fc = fc_arr[shared_fc_type];
1195 if (!(*fc))
1196 return false;
1197
1198 return true;
1199 }
1200
mlx5r_fs_destroy_fcs(struct mlx5_ib_dev * dev,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX])1201 void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
1202 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])
1203 {
1204 struct mlx5_fc *in_use_fc;
1205 int i;
1206
1207 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1208 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1209 if (!fc_arr[i])
1210 continue;
1211
1212 if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) {
1213 fc_arr[i] = NULL;
1214 continue;
1215 }
1216
1217 mlx5_fc_destroy(dev->mdev, fc_arr[i]);
1218 fc_arr[i] = NULL;
1219 }
1220 }
1221
mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev * dev,u32 port_num,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1222 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
1223 struct mlx5_ib_op_fc *opfc,
1224 enum mlx5_ib_optional_counter_type type)
1225 {
1226 enum mlx5_flow_namespace_type fn_type;
1227 int priority, i, err, spec_num;
1228 struct mlx5_flow_act flow_act = {};
1229 struct mlx5_flow_destination dst;
1230 struct mlx5_flow_namespace *ns;
1231 struct mlx5_ib_flow_prio *prio;
1232 struct mlx5_flow_spec *spec;
1233
1234 spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1235 if (!spec)
1236 return -ENOMEM;
1237
1238 switch (type) {
1239 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
1240 if (set_ecn_ce_spec(dev, port_num, &spec[0],
1241 MLX5_FS_IPV4_VERSION) ||
1242 set_ecn_ce_spec(dev, port_num, &spec[1],
1243 MLX5_FS_IPV6_VERSION)) {
1244 err = -EOPNOTSUPP;
1245 goto free;
1246 }
1247 spec_num = 2;
1248 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1249 priority = RDMA_RX_ECN_OPCOUNTER_PRIO;
1250 break;
1251
1252 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
1253 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1254 ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1255 set_cnp_spec(dev, port_num, &spec[0])) {
1256 err = -EOPNOTSUPP;
1257 goto free;
1258 }
1259 spec_num = 1;
1260 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1261 priority = RDMA_RX_CNP_OPCOUNTER_PRIO;
1262 break;
1263
1264 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
1265 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1266 ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1267 set_cnp_spec(dev, port_num, &spec[0])) {
1268 err = -EOPNOTSUPP;
1269 goto free;
1270 }
1271 spec_num = 1;
1272 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1273 priority = RDMA_TX_CNP_OPCOUNTER_PRIO;
1274 break;
1275
1276 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
1277 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
1278 spec_num = 1;
1279 fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1280 priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;
1281 break;
1282
1283 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
1284 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
1285 spec_num = 1;
1286 fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1287 priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;
1288 break;
1289
1290 default:
1291 err = -EOPNOTSUPP;
1292 goto free;
1293 }
1294
1295 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
1296 if (!ns) {
1297 err = -EOPNOTSUPP;
1298 goto free;
1299 }
1300
1301 prio = get_opfc_prio(dev, type);
1302 if (!prio->flow_table) {
1303 err = get_per_qp_prio(dev, type);
1304 if (err)
1305 goto free;
1306
1307 prio = _get_prio(dev, ns, prio, priority,
1308 dev->num_ports * MAX_OPFC_RULES, 1, 0, 0);
1309 if (IS_ERR(prio)) {
1310 err = PTR_ERR(prio);
1311 goto put_prio;
1312 }
1313 }
1314
1315 dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1316 dst.counter = opfc->fc;
1317
1318 flow_act.action =
1319 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1320
1321 for (i = 0; i < spec_num; i++) {
1322 opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1323 &flow_act, &dst, 1);
1324 if (IS_ERR(opfc->rule[i])) {
1325 err = PTR_ERR(opfc->rule[i]);
1326 goto del_rules;
1327 }
1328 }
1329 prio->refcount += spec_num;
1330 kfree(spec);
1331
1332 return 0;
1333
1334 del_rules:
1335 for (i -= 1; i >= 0; i--)
1336 mlx5_del_flow_rules(opfc->rule[i]);
1337 put_flow_table(dev, prio, false);
1338 put_prio:
1339 put_per_qp_prio(dev, type);
1340 free:
1341 kfree(spec);
1342 return err;
1343 }
1344
mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev * dev,struct mlx5_ib_op_fc * opfc,enum mlx5_ib_optional_counter_type type)1345 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
1346 struct mlx5_ib_op_fc *opfc,
1347 enum mlx5_ib_optional_counter_type type)
1348 {
1349 struct mlx5_ib_flow_prio *prio;
1350 int i;
1351
1352 prio = get_opfc_prio(dev, type);
1353
1354 for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) {
1355 mlx5_del_flow_rules(opfc->rule[i]);
1356 put_flow_table(dev, prio, true);
1357 }
1358
1359 put_per_qp_prio(dev, type);
1360 }
1361
mlx5r_fs_unbind_op_fc(struct ib_qp * qp,struct xarray * qpn_opfc_xa)1362 void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)
1363 {
1364 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1365 struct mlx5_per_qp_opfc *per_qp_opfc;
1366 struct mlx5_ib_op_fc *in_use_opfc;
1367 struct mlx5_ib_flow_prio *prio;
1368 int i, j;
1369
1370 per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num);
1371 if (!per_qp_opfc)
1372 return;
1373
1374 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1375 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1376 if (!per_qp_opfc->opfcs[i].fc)
1377 continue;
1378
1379 if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i,
1380 &in_use_opfc)) {
1381 per_qp_opfc->opfcs[i].fc = NULL;
1382 continue;
1383 }
1384
1385 for (j = 0; j < MAX_OPFC_RULES; j++) {
1386 if (!per_qp_opfc->opfcs[i].rule[j])
1387 continue;
1388 mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]);
1389 prio = get_opfc_prio(dev, i);
1390 put_flow_table(dev, prio, true);
1391 }
1392 per_qp_opfc->opfcs[i].fc = NULL;
1393 }
1394
1395 kfree(per_qp_opfc);
1396 xa_erase(qpn_opfc_xa, qp->qp_num);
1397 }
1398
mlx5r_fs_bind_op_fc(struct ib_qp * qp,struct mlx5_fc * fc_arr[MLX5_IB_OPCOUNTER_MAX],struct xarray * qpn_opfc_xa,u32 port)1399 int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
1400 struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1401 struct xarray *qpn_opfc_xa, u32 port)
1402 {
1403 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1404 struct mlx5_per_qp_opfc *per_qp_opfc;
1405 struct mlx5_ib_flow_prio *prio;
1406 struct mlx5_ib_counters *cnts;
1407 struct mlx5_ib_op_fc *opfc;
1408 struct mlx5_fc *in_use_fc;
1409 int i, err, per_qp_type;
1410 bool new;
1411
1412 cnts = &dev->port[port - 1].cnts;
1413
1414 for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
1415 opfc = &cnts->opfcs[i];
1416 if (!opfc->fc)
1417 continue;
1418
1419 per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1420 prio = get_opfc_prio(dev, per_qp_type);
1421 WARN_ON(!prio->flow_table);
1422
1423 if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc))
1424 fc_arr[per_qp_type] = in_use_fc;
1425
1426 if (!fc_arr[per_qp_type]) {
1427 fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false);
1428 if (IS_ERR(fc_arr[per_qp_type]))
1429 return PTR_ERR(fc_arr[per_qp_type]);
1430 }
1431
1432 per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new);
1433 if (!per_qp_opfc) {
1434 err = -ENOMEM;
1435 goto free_fc;
1436 }
1437 err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,
1438 prio, per_qp_type, qp->qp_num, port);
1439 if (err)
1440 goto del_rules;
1441 }
1442
1443 return 0;
1444
1445 del_rules:
1446 mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);
1447 if (new)
1448 kfree(per_qp_opfc);
1449 free_fc:
1450 if (xa_empty(qpn_opfc_xa))
1451 mlx5r_fs_destroy_fcs(dev, fc_arr);
1452 return err;
1453 }
1454
set_underlay_qp(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,u32 underlay_qpn)1455 static void set_underlay_qp(struct mlx5_ib_dev *dev,
1456 struct mlx5_flow_spec *spec,
1457 u32 underlay_qpn)
1458 {
1459 void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
1460 spec->match_criteria,
1461 misc_parameters);
1462 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1463 misc_parameters);
1464
1465 if (underlay_qpn &&
1466 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1467 ft_field_support.bth_dst_qp)) {
1468 MLX5_SET(fte_match_set_misc,
1469 misc_params_v, bth_dst_qp, underlay_qpn);
1470 MLX5_SET(fte_match_set_misc,
1471 misc_params_c, bth_dst_qp, 0xffffff);
1472 }
1473 }
1474
mlx5_ib_set_rule_source_port(struct mlx5_ib_dev * dev,struct mlx5_flow_spec * spec,struct mlx5_eswitch_rep * rep)1475 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
1476 struct mlx5_flow_spec *spec,
1477 struct mlx5_eswitch_rep *rep)
1478 {
1479 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
1480 void *misc;
1481
1482 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1483 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1484 misc_parameters_2);
1485
1486 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1487 mlx5_eswitch_get_vport_metadata_for_match(rep->esw,
1488 rep->vport));
1489 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1490 misc_parameters_2);
1491
1492 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1493 mlx5_eswitch_get_vport_metadata_mask());
1494 } else {
1495 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1496 misc_parameters);
1497
1498 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
1499
1500 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1501 misc_parameters);
1502
1503 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1504 }
1505 }
1506
_create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst,u32 underlay_qpn,struct mlx5_ib_create_flow * ucmd)1507 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
1508 struct mlx5_ib_flow_prio *ft_prio,
1509 const struct ib_flow_attr *flow_attr,
1510 struct mlx5_flow_destination *dst,
1511 u32 underlay_qpn,
1512 struct mlx5_ib_create_flow *ucmd)
1513 {
1514 struct mlx5_flow_table *ft = ft_prio->flow_table;
1515 struct mlx5_ib_flow_handler *handler;
1516 struct mlx5_flow_act flow_act = {};
1517 struct mlx5_flow_spec *spec;
1518 struct mlx5_flow_destination dest_arr[2] = {};
1519 struct mlx5_flow_destination *rule_dst = dest_arr;
1520 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
1521 unsigned int spec_index;
1522 u32 prev_type = 0;
1523 int err = 0;
1524 int dest_num = 0;
1525 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1526
1527 if (!is_valid_attr(dev->mdev, flow_attr))
1528 return ERR_PTR(-EINVAL);
1529
1530 if (dev->is_rep && is_egress)
1531 return ERR_PTR(-EINVAL);
1532
1533 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1534 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1535 if (!handler || !spec) {
1536 err = -ENOMEM;
1537 goto free;
1538 }
1539
1540 INIT_LIST_HEAD(&handler->list);
1541
1542 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
1543 err = parse_flow_attr(dev->mdev, spec,
1544 ib_flow, flow_attr, &flow_act,
1545 prev_type);
1546 if (err < 0)
1547 goto free;
1548
1549 prev_type = ((union ib_flow_spec *)ib_flow)->type;
1550 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
1551 }
1552
1553 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
1554 memcpy(&dest_arr[0], dst, sizeof(*dst));
1555 dest_num++;
1556 }
1557
1558 if (!flow_is_multicast_only(flow_attr))
1559 set_underlay_qp(dev, spec, underlay_qpn);
1560
1561 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
1562 struct mlx5_eswitch_rep *rep;
1563
1564 rep = dev->port[flow_attr->port - 1].rep;
1565 if (!rep) {
1566 err = -EINVAL;
1567 goto free;
1568 }
1569
1570 mlx5_ib_set_rule_source_port(dev, spec, rep);
1571 }
1572
1573 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
1574
1575 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1576 struct mlx5_ib_mcounters *mcounters;
1577
1578 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
1579 if (err)
1580 goto free;
1581
1582 mcounters = to_mcounters(flow_act.counters);
1583 handler->ibcounters = flow_act.counters;
1584 dest_arr[dest_num].type =
1585 MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1586 dest_arr[dest_num].counter =
1587 mcounters->hw_cntrs_hndl;
1588 dest_num++;
1589 }
1590
1591 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1592 if (!dest_num)
1593 rule_dst = NULL;
1594 } else {
1595 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
1596 flow_act.action |=
1597 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1598 if (is_egress)
1599 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1600 else if (dest_num)
1601 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1602 }
1603
1604 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
1605 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1606 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1607 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1608 spec->flow_context.flow_tag, flow_attr->type);
1609 err = -EINVAL;
1610 goto free;
1611 }
1612 handler->rule = mlx5_add_flow_rules(ft, spec,
1613 &flow_act,
1614 rule_dst, dest_num);
1615
1616 if (IS_ERR(handler->rule)) {
1617 err = PTR_ERR(handler->rule);
1618 goto free;
1619 }
1620
1621 ft_prio->refcount++;
1622 handler->prio = ft_prio;
1623 handler->dev = dev;
1624
1625 ft_prio->flow_table = ft;
1626 free:
1627 if (err && handler) {
1628 mlx5_ib_counters_clear_description(handler->ibcounters);
1629 kfree(handler);
1630 }
1631 kvfree(spec);
1632 return err ? ERR_PTR(err) : handler;
1633 }
1634
create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1635 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1636 struct mlx5_ib_flow_prio *ft_prio,
1637 const struct ib_flow_attr *flow_attr,
1638 struct mlx5_flow_destination *dst)
1639 {
1640 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
1641 }
1642
create_leftovers_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)1643 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1644 struct mlx5_ib_flow_prio *ft_prio,
1645 struct ib_flow_attr *flow_attr,
1646 struct mlx5_flow_destination *dst)
1647 {
1648 struct mlx5_ib_flow_handler *handler_ucast = NULL;
1649 struct mlx5_ib_flow_handler *handler = NULL;
1650
1651 static struct {
1652 struct ib_flow_spec_eth eth_flow;
1653 struct ib_flow_attr flow_attr;
1654 } leftovers_wc = { .flow_attr = { .num_of_specs = 1,
1655 .size = sizeof(leftovers_wc) },
1656 .eth_flow = {
1657 .type = IB_FLOW_SPEC_ETH,
1658 .size = sizeof(struct ib_flow_spec_eth),
1659 .mask = { .dst_mac = { 0x1 } },
1660 .val = { .dst_mac = { 0x1 } } } };
1661
1662 static struct {
1663 struct ib_flow_spec_eth eth_flow;
1664 struct ib_flow_attr flow_attr;
1665 } leftovers_uc = { .flow_attr = { .num_of_specs = 1,
1666 .size = sizeof(leftovers_uc) },
1667 .eth_flow = {
1668 .type = IB_FLOW_SPEC_ETH,
1669 .size = sizeof(struct ib_flow_spec_eth),
1670 .mask = { .dst_mac = { 0x1 } },
1671 .val = { .dst_mac = {} } } };
1672
1673 handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
1674 if (!IS_ERR(handler) &&
1675 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1676 handler_ucast = create_flow_rule(dev, ft_prio,
1677 &leftovers_uc.flow_attr, dst);
1678 if (IS_ERR(handler_ucast)) {
1679 mlx5_del_flow_rules(handler->rule);
1680 ft_prio->refcount--;
1681 kfree(handler);
1682 handler = handler_ucast;
1683 } else {
1684 list_add(&handler_ucast->list, &handler->list);
1685 }
1686 }
1687
1688 return handler;
1689 }
1690
create_sniffer_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_rx,struct mlx5_ib_flow_prio * ft_tx,struct mlx5_flow_destination * dst)1691 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1692 struct mlx5_ib_flow_prio *ft_rx,
1693 struct mlx5_ib_flow_prio *ft_tx,
1694 struct mlx5_flow_destination *dst)
1695 {
1696 struct mlx5_ib_flow_handler *handler_rx;
1697 struct mlx5_ib_flow_handler *handler_tx;
1698 int err;
1699 static const struct ib_flow_attr flow_attr = {
1700 .num_of_specs = 0,
1701 .type = IB_FLOW_ATTR_SNIFFER,
1702 .size = sizeof(flow_attr)
1703 };
1704
1705 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
1706 if (IS_ERR(handler_rx)) {
1707 err = PTR_ERR(handler_rx);
1708 goto err;
1709 }
1710
1711 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
1712 if (IS_ERR(handler_tx)) {
1713 err = PTR_ERR(handler_tx);
1714 goto err_tx;
1715 }
1716
1717 list_add(&handler_tx->list, &handler_rx->list);
1718
1719 return handler_rx;
1720
1721 err_tx:
1722 mlx5_del_flow_rules(handler_rx->rule);
1723 ft_rx->refcount--;
1724 kfree(handler_rx);
1725 err:
1726 return ERR_PTR(err);
1727 }
1728
mlx5_ib_create_flow(struct ib_qp * qp,struct ib_flow_attr * flow_attr,struct ib_udata * udata)1729 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1730 struct ib_flow_attr *flow_attr,
1731 struct ib_udata *udata)
1732 {
1733 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1734 struct mlx5_ib_qp *mqp = to_mqp(qp);
1735 struct mlx5_ib_flow_handler *handler = NULL;
1736 struct mlx5_flow_destination *dst = NULL;
1737 struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1738 struct mlx5_ib_flow_prio *ft_prio;
1739 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1740 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1741 size_t min_ucmd_sz, required_ucmd_sz;
1742 int err;
1743 int underlay_qpn;
1744
1745 if (udata && udata->inlen) {
1746 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1747 if (udata->inlen < min_ucmd_sz)
1748 return ERR_PTR(-EOPNOTSUPP);
1749
1750 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
1751 if (err)
1752 return ERR_PTR(err);
1753
1754 /* currently supports only one counters data */
1755 if (ucmd_hdr.ncounters_data > 1)
1756 return ERR_PTR(-EINVAL);
1757
1758 required_ucmd_sz = min_ucmd_sz +
1759 sizeof(struct mlx5_ib_flow_counters_data) *
1760 ucmd_hdr.ncounters_data;
1761 if (udata->inlen > required_ucmd_sz &&
1762 !ib_is_udata_cleared(udata, required_ucmd_sz,
1763 udata->inlen - required_ucmd_sz))
1764 return ERR_PTR(-EOPNOTSUPP);
1765
1766 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1767 if (!ucmd)
1768 return ERR_PTR(-ENOMEM);
1769
1770 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
1771 if (err)
1772 goto free_ucmd;
1773 }
1774
1775 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1776 err = -ENOMEM;
1777 goto free_ucmd;
1778 }
1779
1780 if (flow_attr->flags &
1781 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {
1782 err = -EINVAL;
1783 goto free_ucmd;
1784 }
1785
1786 if (is_egress &&
1787 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1788 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1789 err = -EINVAL;
1790 goto free_ucmd;
1791 }
1792
1793 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1794 if (!dst) {
1795 err = -ENOMEM;
1796 goto free_ucmd;
1797 }
1798
1799 mutex_lock(&dev->flow_db->lock);
1800
1801 ft_prio = get_flow_table(dev, flow_attr,
1802 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1803 if (IS_ERR(ft_prio)) {
1804 err = PTR_ERR(ft_prio);
1805 goto unlock;
1806 }
1807 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1808 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
1809 if (IS_ERR(ft_prio_tx)) {
1810 err = PTR_ERR(ft_prio_tx);
1811 ft_prio_tx = NULL;
1812 goto destroy_ft;
1813 }
1814 }
1815
1816 if (is_egress) {
1817 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1818 } else {
1819 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1820 if (mqp->is_rss)
1821 dst->tir_num = mqp->rss_qp.tirn;
1822 else
1823 dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1824 }
1825
1826 switch (flow_attr->type) {
1827 case IB_FLOW_ATTR_NORMAL:
1828 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1829 mqp->underlay_qpn :
1830 0;
1831 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1832 underlay_qpn, ucmd);
1833 break;
1834 case IB_FLOW_ATTR_ALL_DEFAULT:
1835 case IB_FLOW_ATTR_MC_DEFAULT:
1836 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1837 break;
1838 case IB_FLOW_ATTR_SNIFFER:
1839 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
1840 break;
1841 default:
1842 err = -EINVAL;
1843 goto destroy_ft;
1844 }
1845
1846 if (IS_ERR(handler)) {
1847 err = PTR_ERR(handler);
1848 handler = NULL;
1849 goto destroy_ft;
1850 }
1851
1852 mutex_unlock(&dev->flow_db->lock);
1853 kfree(dst);
1854 kfree(ucmd);
1855
1856 return &handler->ibflow;
1857
1858 destroy_ft:
1859 put_flow_table(dev, ft_prio, false);
1860 if (ft_prio_tx)
1861 put_flow_table(dev, ft_prio_tx, false);
1862 unlock:
1863 mutex_unlock(&dev->flow_db->lock);
1864 kfree(dst);
1865 free_ucmd:
1866 kfree(ucmd);
1867 return ERR_PTR(err);
1868 }
1869
mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev * dev,enum mlx5_flow_namespace_type type,u32 * flags,u16 * vport_idx,u16 * vport,struct mlx5_core_dev ** ft_mdev,u32 ib_port)1870 static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,
1871 enum mlx5_flow_namespace_type type,
1872 u32 *flags, u16 *vport_idx,
1873 u16 *vport,
1874 struct mlx5_core_dev **ft_mdev,
1875 u32 ib_port)
1876 {
1877 struct mlx5_core_dev *esw_mdev;
1878
1879 if (!is_mdev_switchdev_mode(dev->mdev))
1880 return 0;
1881
1882 if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager))
1883 return -EOPNOTSUPP;
1884
1885 if (!dev->port[ib_port - 1].rep)
1886 return -EINVAL;
1887
1888 esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw);
1889 if (esw_mdev != dev->mdev)
1890 return -EOPNOTSUPP;
1891
1892 *flags |= MLX5_FLOW_TABLE_OTHER_VPORT;
1893 *ft_mdev = esw_mdev;
1894 *vport = dev->port[ib_port - 1].rep->vport;
1895 *vport_idx = dev->port[ib_port - 1].rep->vport_index;
1896
1897 return 0;
1898 }
1899
1900 static struct mlx5_ib_flow_prio *
_get_flow_table(struct mlx5_ib_dev * dev,u16 user_priority,enum mlx5_flow_namespace_type ns_type,bool mcast,u32 ib_port)1901 _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
1902 enum mlx5_flow_namespace_type ns_type,
1903 bool mcast, u32 ib_port)
1904 {
1905 struct mlx5_core_dev *ft_mdev = dev->mdev;
1906 struct mlx5_flow_namespace *ns = NULL;
1907 struct mlx5_ib_flow_prio *prio = NULL;
1908 int max_table_size = 0;
1909 u16 vport_idx = 0;
1910 bool esw_encap;
1911 u32 flags = 0;
1912 u16 vport = 0;
1913 int priority;
1914 int ret;
1915
1916 if (mcast)
1917 priority = MLX5_IB_FLOW_MCAST_PRIO;
1918 else
1919 priority = ib_prio_to_core_prio(user_priority, false);
1920
1921 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
1922 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1923 switch (ns_type) {
1924 case MLX5_FLOW_NAMESPACE_BYPASS:
1925 max_table_size = BIT(
1926 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1927 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1928 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1929 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1930 reformat_l3_tunnel_to_l2) &&
1931 !esw_encap)
1932 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1933 break;
1934 case MLX5_FLOW_NAMESPACE_EGRESS:
1935 max_table_size = BIT(
1936 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1937 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1938 !esw_encap)
1939 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1940 break;
1941 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
1942 max_table_size = BIT(
1943 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1944 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1945 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1946 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1947 reformat_l3_tunnel_to_l2) &&
1948 esw_encap)
1949 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1950 priority = user_priority;
1951 break;
1952 case MLX5_FLOW_NAMESPACE_RDMA_RX:
1953 max_table_size = BIT(
1954 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1955 priority = user_priority;
1956 break;
1957 case MLX5_FLOW_NAMESPACE_RDMA_TX:
1958 max_table_size = BIT(
1959 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1960 priority = user_priority;
1961 break;
1962 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
1963 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
1964 if (ib_port == 0 ||
1965 user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
1966 return ERR_PTR(-EINVAL);
1967 ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags,
1968 &vport_idx, &vport,
1969 &ft_mdev, ib_port);
1970 if (ret)
1971 return ERR_PTR(ret);
1972
1973 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)
1974 max_table_size =
1975 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(
1976 ft_mdev, log_max_ft_size));
1977 else
1978 max_table_size =
1979 BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(
1980 ft_mdev, log_max_ft_size));
1981 priority = user_priority;
1982 break;
1983 default:
1984 break;
1985 }
1986
1987 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1988
1989 if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||
1990 ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)
1991 ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx);
1992 else
1993 ns = mlx5_get_flow_namespace(ft_mdev, ns_type);
1994
1995 if (!ns)
1996 return ERR_PTR(-EOPNOTSUPP);
1997
1998 switch (ns_type) {
1999 case MLX5_FLOW_NAMESPACE_BYPASS:
2000 prio = &dev->flow_db->prios[priority];
2001 break;
2002 case MLX5_FLOW_NAMESPACE_EGRESS:
2003 prio = &dev->flow_db->egress_prios[priority];
2004 break;
2005 case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
2006 prio = &dev->flow_db->fdb[priority];
2007 break;
2008 case MLX5_FLOW_NAMESPACE_RDMA_RX:
2009 prio = &dev->flow_db->rdma_rx[priority];
2010 break;
2011 case MLX5_FLOW_NAMESPACE_RDMA_TX:
2012 prio = &dev->flow_db->rdma_tx[priority];
2013 break;
2014 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
2015 prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1];
2016 break;
2017 case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
2018 prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1];
2019 break;
2020 default: return ERR_PTR(-EINVAL);
2021 }
2022
2023 if (!prio)
2024 return ERR_PTR(-EINVAL);
2025
2026 if (prio->flow_table)
2027 return prio;
2028
2029 return _get_prio(dev, ns, prio, priority, max_table_size,
2030 MLX5_FS_MAX_TYPES, flags, vport);
2031 }
2032
2033 static struct mlx5_ib_flow_handler *
_create_raw_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct mlx5_flow_destination * dst,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,void * cmd_in,int inlen,int dst_num)2034 _create_raw_flow_rule(struct mlx5_ib_dev *dev,
2035 struct mlx5_ib_flow_prio *ft_prio,
2036 struct mlx5_flow_destination *dst,
2037 struct mlx5_ib_flow_matcher *fs_matcher,
2038 struct mlx5_flow_context *flow_context,
2039 struct mlx5_flow_act *flow_act,
2040 void *cmd_in, int inlen,
2041 int dst_num)
2042 {
2043 struct mlx5_ib_flow_handler *handler;
2044 struct mlx5_flow_spec *spec;
2045 struct mlx5_flow_table *ft = ft_prio->flow_table;
2046 int err = 0;
2047
2048 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2049 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2050 if (!handler || !spec) {
2051 err = -ENOMEM;
2052 goto free;
2053 }
2054
2055 INIT_LIST_HEAD(&handler->list);
2056
2057 memcpy(spec->match_value, cmd_in, inlen);
2058 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
2059 fs_matcher->mask_len);
2060 spec->match_criteria_enable = fs_matcher->match_criteria_enable;
2061 spec->flow_context = *flow_context;
2062
2063 handler->rule = mlx5_add_flow_rules(ft, spec,
2064 flow_act, dst, dst_num);
2065
2066 if (IS_ERR(handler->rule)) {
2067 err = PTR_ERR(handler->rule);
2068 goto free;
2069 }
2070
2071 ft_prio->refcount++;
2072 handler->prio = ft_prio;
2073 handler->dev = dev;
2074 ft_prio->flow_table = ft;
2075
2076 free:
2077 if (err)
2078 kfree(handler);
2079 kvfree(spec);
2080 return err ? ERR_PTR(err) : handler;
2081 }
2082
raw_fs_is_multicast(struct mlx5_ib_flow_matcher * fs_matcher,void * match_v)2083 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
2084 void *match_v)
2085 {
2086 void *match_c;
2087 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
2088 void *dmac, *dmac_mask;
2089 void *ipv4, *ipv4_mask;
2090
2091 if (!(fs_matcher->match_criteria_enable &
2092 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
2093 return false;
2094
2095 match_c = fs_matcher->matcher_mask.match_params;
2096 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
2097 outer_headers);
2098 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
2099 outer_headers);
2100
2101 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2102 dmac_47_16);
2103 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2104 dmac_47_16);
2105
2106 if (is_multicast_ether_addr(dmac) &&
2107 is_multicast_ether_addr(dmac_mask))
2108 return true;
2109
2110 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2111 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2112
2113 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2114 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2115
2116 if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
2117 ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
2118 return true;
2119
2120 return false;
2121 }
2122
raw_fs_rule_add(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_matcher * fs_matcher,struct mlx5_flow_context * flow_context,struct mlx5_flow_act * flow_act,struct mlx5_fc * counter,void * cmd_in,int inlen,int dest_id,int dest_type)2123 static struct mlx5_ib_flow_handler *raw_fs_rule_add(
2124 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
2125 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
2126 struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)
2127 {
2128 struct mlx5_flow_destination *dst;
2129 struct mlx5_ib_flow_prio *ft_prio;
2130 struct mlx5_ib_flow_handler *handler;
2131 int dst_num = 0;
2132 bool mcast;
2133 int err;
2134
2135 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
2136 return ERR_PTR(-EOPNOTSUPP);
2137
2138 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
2139 return ERR_PTR(-ENOMEM);
2140
2141 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
2142 if (!dst)
2143 return ERR_PTR(-ENOMEM);
2144
2145 mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
2146 mutex_lock(&dev->flow_db->lock);
2147
2148 ft_prio = _get_flow_table(dev, fs_matcher->priority,
2149 fs_matcher->ns_type, mcast,
2150 fs_matcher->ib_port);
2151 if (IS_ERR(ft_prio)) {
2152 err = PTR_ERR(ft_prio);
2153 goto unlock;
2154 }
2155
2156 switch (dest_type) {
2157 case MLX5_FLOW_DESTINATION_TYPE_TIR:
2158 dst[dst_num].type = dest_type;
2159 dst[dst_num++].tir_num = dest_id;
2160 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2161 break;
2162 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
2163 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
2164 dst[dst_num++].ft_num = dest_id;
2165 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2166 break;
2167 case MLX5_FLOW_DESTINATION_TYPE_PORT:
2168 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2169 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2170 break;
2171 default:
2172 break;
2173 }
2174
2175 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
2176 if (WARN_ON(!counter)) {
2177 err = -EINVAL;
2178 goto unlock;
2179 }
2180 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2181 dst[dst_num].counter = counter;
2182 dst_num++;
2183 }
2184
2185 handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,
2186 fs_matcher, flow_context, flow_act,
2187 cmd_in, inlen, dst_num);
2188
2189 if (IS_ERR(handler)) {
2190 err = PTR_ERR(handler);
2191 goto destroy_ft;
2192 }
2193
2194 mutex_unlock(&dev->flow_db->lock);
2195 atomic_inc(&fs_matcher->usecnt);
2196 handler->flow_matcher = fs_matcher;
2197
2198 kfree(dst);
2199
2200 return handler;
2201
2202 destroy_ft:
2203 put_flow_table(dev, ft_prio, false);
2204 unlock:
2205 mutex_unlock(&dev->flow_db->lock);
2206 kfree(dst);
2207
2208 return ERR_PTR(err);
2209 }
2210
destroy_flow_action_raw(struct mlx5_ib_flow_action * maction)2211 static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
2212 {
2213 switch (maction->flow_action_raw.sub_type) {
2214 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
2215 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
2216 maction->flow_action_raw.modify_hdr);
2217 break;
2218 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
2219 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
2220 maction->flow_action_raw.pkt_reformat);
2221 break;
2222 case MLX5_IB_FLOW_ACTION_DECAP:
2223 break;
2224 default:
2225 break;
2226 }
2227 }
2228
mlx5_ib_destroy_flow_action(struct ib_flow_action * action)2229 static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
2230 {
2231 struct mlx5_ib_flow_action *maction = to_mflow_act(action);
2232
2233 switch (action->type) {
2234 case IB_FLOW_ACTION_UNSPECIFIED:
2235 destroy_flow_action_raw(maction);
2236 break;
2237 default:
2238 WARN_ON(true);
2239 break;
2240 }
2241
2242 kfree(maction);
2243 return 0;
2244 }
2245
2246 static int
mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,enum mlx5_flow_namespace_type * namespace)2247 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
2248 enum mlx5_flow_namespace_type *namespace)
2249 {
2250 switch (table_type) {
2251 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
2252 *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
2253 break;
2254 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
2255 *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
2256 break;
2257 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
2258 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;
2259 break;
2260 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
2261 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
2262 break;
2263 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
2264 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
2265 break;
2266 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:
2267 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;
2268 break;
2269 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:
2270 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;
2271 break;
2272 default:
2273 return -EINVAL;
2274 }
2275
2276 return 0;
2277 }
2278
2279 static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
2280 [MLX5_IB_FLOW_TYPE_NORMAL] = {
2281 .type = UVERBS_ATTR_TYPE_PTR_IN,
2282 .u.ptr = {
2283 .len = sizeof(u16), /* data is priority */
2284 .min_len = sizeof(u16),
2285 }
2286 },
2287 [MLX5_IB_FLOW_TYPE_SNIFFER] = {
2288 .type = UVERBS_ATTR_TYPE_PTR_IN,
2289 UVERBS_ATTR_NO_DATA(),
2290 },
2291 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
2292 .type = UVERBS_ATTR_TYPE_PTR_IN,
2293 UVERBS_ATTR_NO_DATA(),
2294 },
2295 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
2296 .type = UVERBS_ATTR_TYPE_PTR_IN,
2297 UVERBS_ATTR_NO_DATA(),
2298 },
2299 };
2300
is_flow_dest(void * obj,int * dest_id,int * dest_type)2301 static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
2302 {
2303 struct devx_obj *devx_obj = obj;
2304 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2305
2306 switch (opcode) {
2307 case MLX5_CMD_OP_DESTROY_TIR:
2308 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2309 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
2310 obj_id);
2311 return true;
2312
2313 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
2314 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2315 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
2316 table_id);
2317 return true;
2318 default:
2319 return false;
2320 }
2321 }
2322
get_dests(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * fs_matcher,int * dest_id,int * dest_type,struct ib_qp ** qp,u32 * flags)2323 static int get_dests(struct uverbs_attr_bundle *attrs,
2324 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
2325 int *dest_type, struct ib_qp **qp, u32 *flags)
2326 {
2327 bool dest_devx, dest_qp;
2328 void *devx_obj;
2329 int err;
2330
2331 dest_devx = uverbs_attr_is_valid(attrs,
2332 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2333 dest_qp = uverbs_attr_is_valid(attrs,
2334 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2335
2336 *flags = 0;
2337 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2338 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
2339 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
2340 if (err)
2341 return err;
2342
2343 /* Both flags are not allowed */
2344 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
2345 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2346 return -EINVAL;
2347
2348 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
2349 if (dest_devx && (dest_qp || *flags))
2350 return -EINVAL;
2351 else if (dest_qp && *flags)
2352 return -EINVAL;
2353 }
2354
2355 /* Allow only DEVX object, drop as dest for FDB */
2356 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2357 !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
2358 return -EINVAL;
2359
2360 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
2361 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2362 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2363 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
2364 return -EINVAL;
2365
2366 *qp = NULL;
2367 if (dest_devx) {
2368 devx_obj =
2369 uverbs_attr_get_obj(attrs,
2370 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2371
2372 /* Verify that the given DEVX object is a flow
2373 * steering destination.
2374 */
2375 if (!is_flow_dest(devx_obj, dest_id, dest_type))
2376 return -EINVAL;
2377 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
2378 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||
2379 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2380 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2381 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
2382 return -EINVAL;
2383 } else if (dest_qp) {
2384 struct mlx5_ib_qp *mqp;
2385
2386 *qp = uverbs_attr_get_obj(attrs,
2387 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2388 if (IS_ERR(*qp))
2389 return PTR_ERR(*qp);
2390
2391 if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
2392 return -EINVAL;
2393
2394 mqp = to_mqp(*qp);
2395 if (mqp->is_rss)
2396 *dest_id = mqp->rss_qp.tirn;
2397 else
2398 *dest_id = mqp->raw_packet_qp.rq.tirn;
2399 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2400 } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2401 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2402 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &&
2403 !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {
2404 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2405 }
2406
2407 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
2408 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2409 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2410 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))
2411 return -EINVAL;
2412
2413 return 0;
2414 }
2415
2416 static bool
is_flow_counter(void * obj,u32 offset,u32 * counter_id,u32 * fc_bulk_size)2417 is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)
2418 {
2419 struct devx_obj *devx_obj = obj;
2420 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2421
2422 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
2423
2424 if (offset && offset >= devx_obj->flow_counter_bulk_size)
2425 return false;
2426
2427 *fc_bulk_size = devx_obj->flow_counter_bulk_size;
2428 *counter_id = MLX5_GET(dealloc_flow_counter_in,
2429 devx_obj->dinbox,
2430 flow_counter_id);
2431 *counter_id += offset;
2432 return true;
2433 }
2434
2435 return false;
2436 }
2437
2438 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)2439 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
2440 struct uverbs_attr_bundle *attrs)
2441 {
2442 struct mlx5_flow_context flow_context = {.flow_tag =
2443 MLX5_FS_DEFAULT_FLOW_TAG};
2444 int dest_id, dest_type = -1, inlen, len, ret, i;
2445 struct mlx5_ib_flow_handler *flow_handler;
2446 struct mlx5_ib_flow_matcher *fs_matcher;
2447 struct ib_uobject **arr_flow_actions;
2448 struct ib_uflow_resources *uflow_res;
2449 struct mlx5_flow_act flow_act = {};
2450 struct mlx5_fc *counter = NULL;
2451 struct ib_qp *qp = NULL;
2452 void *devx_obj, *cmd_in;
2453 struct ib_uobject *uobj;
2454 struct mlx5_ib_dev *dev;
2455 u32 flags;
2456
2457 if (!rdma_uattrs_has_raw_cap(attrs))
2458 return -EPERM;
2459
2460 fs_matcher = uverbs_attr_get_obj(attrs,
2461 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
2462 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
2463 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2464
2465 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
2466 return -EINVAL;
2467
2468 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
2469 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
2470
2471 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2472 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2473
2474 len = uverbs_attr_get_uobjs_arr(attrs,
2475 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
2476 if (len) {
2477 u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;
2478 devx_obj = arr_flow_actions[0]->object;
2479
2480 if (uverbs_attr_is_valid(attrs,
2481 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
2482
2483 int num_offsets = uverbs_attr_ptr_get_array_size(
2484 attrs,
2485 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2486 sizeof(u32));
2487
2488 if (num_offsets != 1)
2489 return -EINVAL;
2490
2491 offset_attr = uverbs_attr_get_alloced_ptr(
2492 attrs,
2493 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
2494 offset = *offset_attr;
2495 }
2496
2497 if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size))
2498 return -EINVAL;
2499 counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size);
2500 if (IS_ERR(counter))
2501 return PTR_ERR(counter);
2502
2503 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2504 }
2505
2506 cmd_in = uverbs_attr_get_alloced_ptr(
2507 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2508 inlen = uverbs_attr_get_len(attrs,
2509 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2510
2511 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
2512 if (!uflow_res) {
2513 ret = -ENOMEM;
2514 goto destroy_counter;
2515 }
2516
2517 len = uverbs_attr_get_uobjs_arr(attrs,
2518 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
2519 for (i = 0; i < len; i++) {
2520 struct mlx5_ib_flow_action *maction =
2521 to_mflow_act(arr_flow_actions[i]->object);
2522
2523 ret = parse_flow_flow_action(maction, false, &flow_act);
2524 if (ret)
2525 goto err_out;
2526 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
2527 arr_flow_actions[i]->object);
2528 }
2529
2530 ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2531 MLX5_IB_ATTR_CREATE_FLOW_TAG);
2532 if (!ret) {
2533 if (flow_context.flow_tag >= BIT(24)) {
2534 ret = -EINVAL;
2535 goto err_out;
2536 }
2537 flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2538 }
2539
2540 flow_handler =
2541 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
2542 counter, cmd_in, inlen, dest_id, dest_type);
2543 if (IS_ERR(flow_handler)) {
2544 ret = PTR_ERR(flow_handler);
2545 goto err_out;
2546 }
2547
2548 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
2549
2550 return 0;
2551 err_out:
2552 ib_uverbs_flow_resources_free(uflow_res);
2553 destroy_counter:
2554 if (counter)
2555 mlx5_fc_local_destroy(counter);
2556 return ret;
2557 }
2558
flow_matcher_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2559 static int flow_matcher_cleanup(struct ib_uobject *uobject,
2560 enum rdma_remove_reason why,
2561 struct uverbs_attr_bundle *attrs)
2562 {
2563 struct mlx5_ib_flow_matcher *obj = uobject->object;
2564
2565 if (atomic_read(&obj->usecnt))
2566 return -EBUSY;
2567
2568 kfree(obj);
2569 return 0;
2570 }
2571
steering_anchor_create_ft(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2572 static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
2573 struct mlx5_ib_flow_prio *ft_prio,
2574 enum mlx5_flow_namespace_type ns_type)
2575 {
2576 struct mlx5_flow_table_attr ft_attr = {};
2577 struct mlx5_flow_namespace *ns;
2578 struct mlx5_flow_table *ft;
2579
2580 if (ft_prio->anchor.ft)
2581 return 0;
2582
2583 ns = mlx5_get_flow_namespace(dev->mdev, ns_type);
2584 if (!ns)
2585 return -EOPNOTSUPP;
2586
2587 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
2588 ft_attr.uid = MLX5_SHARED_RESOURCE_UID;
2589 ft_attr.prio = 0;
2590 ft_attr.max_fte = 2;
2591 ft_attr.level = 1;
2592
2593 ft = mlx5_create_flow_table(ns, &ft_attr);
2594 if (IS_ERR(ft))
2595 return PTR_ERR(ft);
2596
2597 ft_prio->anchor.ft = ft;
2598
2599 return 0;
2600 }
2601
steering_anchor_destroy_ft(struct mlx5_ib_flow_prio * ft_prio)2602 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
2603 {
2604 if (ft_prio->anchor.ft) {
2605 mlx5_destroy_flow_table(ft_prio->anchor.ft);
2606 ft_prio->anchor.ft = NULL;
2607 }
2608 }
2609
2610 static int
steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2611 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2612 {
2613 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2614 struct mlx5_flow_group *fg;
2615 void *flow_group_in;
2616 int err = 0;
2617
2618 if (ft_prio->anchor.fg_drop)
2619 return 0;
2620
2621 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2622 if (!flow_group_in)
2623 return -ENOMEM;
2624
2625 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
2626 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
2627
2628 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2629 if (IS_ERR(fg)) {
2630 err = PTR_ERR(fg);
2631 goto out;
2632 }
2633
2634 ft_prio->anchor.fg_drop = fg;
2635
2636 out:
2637 kvfree(flow_group_in);
2638
2639 return err;
2640 }
2641
2642 static void
steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio * ft_prio)2643 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2644 {
2645 if (ft_prio->anchor.fg_drop) {
2646 mlx5_destroy_flow_group(ft_prio->anchor.fg_drop);
2647 ft_prio->anchor.fg_drop = NULL;
2648 }
2649 }
2650
2651 static int
steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2652 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2653 {
2654 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2655 struct mlx5_flow_group *fg;
2656 void *flow_group_in;
2657 int err = 0;
2658
2659 if (ft_prio->anchor.fg_goto_table)
2660 return 0;
2661
2662 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2663 if (!flow_group_in)
2664 return -ENOMEM;
2665
2666 fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2667 if (IS_ERR(fg)) {
2668 err = PTR_ERR(fg);
2669 goto out;
2670 }
2671 ft_prio->anchor.fg_goto_table = fg;
2672
2673 out:
2674 kvfree(flow_group_in);
2675
2676 return err;
2677 }
2678
2679 static void
steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio * ft_prio)2680 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2681 {
2682 if (ft_prio->anchor.fg_goto_table) {
2683 mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table);
2684 ft_prio->anchor.fg_goto_table = NULL;
2685 }
2686 }
2687
2688 static int
steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2689 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2690 {
2691 struct mlx5_flow_act flow_act = {};
2692 struct mlx5_flow_handle *handle;
2693
2694 if (ft_prio->anchor.rule_drop)
2695 return 0;
2696
2697 flow_act.fg = ft_prio->anchor.fg_drop;
2698 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2699
2700 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2701 NULL, 0);
2702 if (IS_ERR(handle))
2703 return PTR_ERR(handle);
2704
2705 ft_prio->anchor.rule_drop = handle;
2706
2707 return 0;
2708 }
2709
steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio * ft_prio)2710 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2711 {
2712 if (ft_prio->anchor.rule_drop) {
2713 mlx5_del_flow_rules(ft_prio->anchor.rule_drop);
2714 ft_prio->anchor.rule_drop = NULL;
2715 }
2716 }
2717
2718 static int
steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2719 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2720 {
2721 struct mlx5_flow_destination dest = {};
2722 struct mlx5_flow_act flow_act = {};
2723 struct mlx5_flow_handle *handle;
2724
2725 if (ft_prio->anchor.rule_goto_table)
2726 return 0;
2727
2728 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2729 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
2730 flow_act.fg = ft_prio->anchor.fg_goto_table;
2731
2732 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2733 dest.ft = ft_prio->flow_table;
2734
2735 handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2736 &dest, 1);
2737 if (IS_ERR(handle))
2738 return PTR_ERR(handle);
2739
2740 ft_prio->anchor.rule_goto_table = handle;
2741
2742 return 0;
2743 }
2744
2745 static void
steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio * ft_prio)2746 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2747 {
2748 if (ft_prio->anchor.rule_goto_table) {
2749 mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table);
2750 ft_prio->anchor.rule_goto_table = NULL;
2751 }
2752 }
2753
steering_anchor_create_res(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,enum mlx5_flow_namespace_type ns_type)2754 static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
2755 struct mlx5_ib_flow_prio *ft_prio,
2756 enum mlx5_flow_namespace_type ns_type)
2757 {
2758 int err;
2759
2760 err = steering_anchor_create_ft(dev, ft_prio, ns_type);
2761 if (err)
2762 return err;
2763
2764 err = steering_anchor_create_fg_drop(ft_prio);
2765 if (err)
2766 goto destroy_ft;
2767
2768 err = steering_anchor_create_fg_goto_table(ft_prio);
2769 if (err)
2770 goto destroy_fg_drop;
2771
2772 err = steering_anchor_create_rule_drop(ft_prio);
2773 if (err)
2774 goto destroy_fg_goto_table;
2775
2776 err = steering_anchor_create_rule_goto_table(ft_prio);
2777 if (err)
2778 goto destroy_rule_drop;
2779
2780 return 0;
2781
2782 destroy_rule_drop:
2783 steering_anchor_destroy_rule_drop(ft_prio);
2784 destroy_fg_goto_table:
2785 steering_anchor_destroy_fg_goto_table(ft_prio);
2786 destroy_fg_drop:
2787 steering_anchor_destroy_fg_drop(ft_prio);
2788 destroy_ft:
2789 steering_anchor_destroy_ft(ft_prio);
2790
2791 return err;
2792 }
2793
mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio * ft_prio)2794 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
2795 {
2796 steering_anchor_destroy_rule_goto_table(ft_prio);
2797 steering_anchor_destroy_rule_drop(ft_prio);
2798 steering_anchor_destroy_fg_goto_table(ft_prio);
2799 steering_anchor_destroy_fg_drop(ft_prio);
2800 steering_anchor_destroy_ft(ft_prio);
2801 }
2802
steering_anchor_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)2803 static int steering_anchor_cleanup(struct ib_uobject *uobject,
2804 enum rdma_remove_reason why,
2805 struct uverbs_attr_bundle *attrs)
2806 {
2807 struct mlx5_ib_steering_anchor *obj = uobject->object;
2808
2809 if (atomic_read(&obj->usecnt))
2810 return -EBUSY;
2811
2812 mutex_lock(&obj->dev->flow_db->lock);
2813 if (!--obj->ft_prio->anchor.rule_goto_table_ref)
2814 steering_anchor_destroy_rule_goto_table(obj->ft_prio);
2815
2816 put_flow_table(obj->dev, obj->ft_prio, true);
2817 mutex_unlock(&obj->dev->flow_db->lock);
2818
2819 kfree(obj);
2820 return 0;
2821 }
2822
fs_cleanup_anchor(struct mlx5_ib_flow_prio * prio,int count)2823 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
2824 int count)
2825 {
2826 while (count--)
2827 mlx5_steering_anchor_destroy_res(&prio[count]);
2828 }
2829
mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev * dev)2830 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
2831 {
2832 fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT);
2833 fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT);
2834 fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS);
2835 fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS);
2836 fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS);
2837 fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT);
2838 fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT);
2839 }
2840
mlx5_ib_matcher_ns(struct uverbs_attr_bundle * attrs,struct mlx5_ib_flow_matcher * obj)2841 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2842 struct mlx5_ib_flow_matcher *obj)
2843 {
2844 enum mlx5_ib_uapi_flow_table_type ft_type =
2845 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2846 u32 flags;
2847 int err;
2848
2849 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2850 * users should switch to it. We leave this to not break userspace
2851 */
2852 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2853 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2854 return -EINVAL;
2855
2856 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2857 err = uverbs_get_const(&ft_type, attrs,
2858 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2859 if (err)
2860 return err;
2861
2862 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
2863 if (err)
2864 return err;
2865
2866 return 0;
2867 }
2868
2869 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2870 err = uverbs_get_flags32(&flags, attrs,
2871 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2872 IB_FLOW_ATTR_FLAGS_EGRESS);
2873 if (err)
2874 return err;
2875
2876 if (flags)
2877 return mlx5_ib_ft_type_to_namespace(
2878 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2879 &obj->ns_type);
2880 }
2881
2882 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2883
2884 return 0;
2885 }
2886
verify_context_caps(struct mlx5_ib_dev * dev,u64 enabled_caps)2887 static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)
2888 {
2889 if (is_mdev_switchdev_mode(dev->mdev))
2890 return UCAP_ENABLED(enabled_caps,
2891 RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
2892
2893 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);
2894 }
2895
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)2896 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2897 struct uverbs_attr_bundle *attrs)
2898 {
2899 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2900 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2901 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2902 struct mlx5_ib_flow_matcher *obj;
2903 int err;
2904
2905 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2906 if (!obj)
2907 return -ENOMEM;
2908
2909 obj->mask_len = uverbs_attr_get_len(
2910 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2911 err = uverbs_copy_from(&obj->matcher_mask,
2912 attrs,
2913 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2914 if (err)
2915 goto end;
2916
2917 obj->flow_type = uverbs_attr_get_enum_id(
2918 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2919
2920 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2921 err = uverbs_copy_from(&obj->priority,
2922 attrs,
2923 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2924 if (err)
2925 goto end;
2926 }
2927
2928 err = uverbs_copy_from(&obj->match_criteria_enable,
2929 attrs,
2930 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2931 if (err)
2932 goto end;
2933
2934 err = mlx5_ib_matcher_ns(attrs, obj);
2935 if (err)
2936 goto end;
2937
2938 if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2939 mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) {
2940 err = -EINVAL;
2941 goto end;
2942 }
2943
2944 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {
2945 err = uverbs_copy_from(&obj->ib_port, attrs,
2946 MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);
2947 if (err)
2948 goto end;
2949 if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) {
2950 err = -EINVAL;
2951 goto end;
2952 }
2953 if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &&
2954 obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {
2955 err = -EINVAL;
2956 goto end;
2957 }
2958 if (!verify_context_caps(dev, uobj->context->enabled_caps)) {
2959 err = -EOPNOTSUPP;
2960 goto end;
2961 }
2962 }
2963
2964 uobj->object = obj;
2965 obj->mdev = dev->mdev;
2966 atomic_set(&obj->usecnt, 0);
2967 return 0;
2968
2969 end:
2970 kfree(obj);
2971 return err;
2972 }
2973
UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)2974 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
2975 struct uverbs_attr_bundle *attrs)
2976 {
2977 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2978 attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);
2979 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2980 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;
2981 enum mlx5_flow_namespace_type ns_type;
2982 struct mlx5_ib_steering_anchor *obj;
2983 struct mlx5_ib_flow_prio *ft_prio;
2984 u16 priority;
2985 u32 ft_id;
2986 int err;
2987
2988 if (!rdma_dev_has_raw_cap(&dev->ib_dev))
2989 return -EPERM;
2990
2991 err = uverbs_get_const(&ib_uapi_ft_type, attrs,
2992 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);
2993 if (err)
2994 return err;
2995
2996 err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type);
2997 if (err)
2998 return err;
2999
3000 err = uverbs_copy_from(&priority, attrs,
3001 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);
3002 if (err)
3003 return err;
3004
3005 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3006 if (!obj)
3007 return -ENOMEM;
3008
3009 mutex_lock(&dev->flow_db->lock);
3010
3011 ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0);
3012 if (IS_ERR(ft_prio)) {
3013 err = PTR_ERR(ft_prio);
3014 goto free_obj;
3015 }
3016
3017 ft_prio->refcount++;
3018
3019 if (!ft_prio->anchor.rule_goto_table_ref) {
3020 err = steering_anchor_create_res(dev, ft_prio, ns_type);
3021 if (err)
3022 goto put_flow_table;
3023 }
3024
3025 ft_prio->anchor.rule_goto_table_ref++;
3026
3027 ft_id = mlx5_flow_table_id(ft_prio->anchor.ft);
3028
3029 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3030 &ft_id, sizeof(ft_id));
3031 if (err)
3032 goto destroy_res;
3033
3034 mutex_unlock(&dev->flow_db->lock);
3035
3036 uobj->object = obj;
3037 obj->dev = dev;
3038 obj->ft_prio = ft_prio;
3039 atomic_set(&obj->usecnt, 0);
3040
3041 return 0;
3042
3043 destroy_res:
3044 --ft_prio->anchor.rule_goto_table_ref;
3045 mlx5_steering_anchor_destroy_res(ft_prio);
3046 put_flow_table:
3047 put_flow_table(dev, ft_prio, true);
3048 free_obj:
3049 mutex_unlock(&dev->flow_db->lock);
3050 kfree(obj);
3051
3052 return err;
3053 }
3054
3055 static struct ib_flow_action *
mlx5_ib_create_modify_header(struct mlx5_ib_dev * dev,enum mlx5_ib_uapi_flow_table_type ft_type,u8 num_actions,void * in)3056 mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
3057 enum mlx5_ib_uapi_flow_table_type ft_type,
3058 u8 num_actions, void *in)
3059 {
3060 enum mlx5_flow_namespace_type namespace;
3061 struct mlx5_ib_flow_action *maction;
3062 int ret;
3063
3064 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3065 if (ret)
3066 return ERR_PTR(-EINVAL);
3067
3068 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3069 if (!maction)
3070 return ERR_PTR(-ENOMEM);
3071
3072 maction->flow_action_raw.modify_hdr =
3073 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
3074
3075 if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
3076 ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
3077 kfree(maction);
3078 return ERR_PTR(ret);
3079 }
3080 maction->flow_action_raw.sub_type =
3081 MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
3082 maction->flow_action_raw.dev = dev;
3083
3084 return &maction->ib_action;
3085 }
3086
mlx5_ib_modify_header_supported(struct mlx5_ib_dev * dev)3087 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
3088 {
3089 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3090 max_modify_header_actions) ||
3091 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3092 max_modify_header_actions) ||
3093 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
3094 max_modify_header_actions);
3095 }
3096
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)3097 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
3098 struct uverbs_attr_bundle *attrs)
3099 {
3100 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3101 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
3102 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3103 enum mlx5_ib_uapi_flow_table_type ft_type;
3104 struct ib_flow_action *action;
3105 int num_actions;
3106 void *in;
3107 int ret;
3108
3109 if (!mlx5_ib_modify_header_supported(mdev))
3110 return -EOPNOTSUPP;
3111
3112 in = uverbs_attr_get_alloced_ptr(attrs,
3113 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
3114
3115 num_actions = uverbs_attr_ptr_get_array_size(
3116 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3117 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
3118 if (num_actions < 0)
3119 return num_actions;
3120
3121 ret = uverbs_get_const(&ft_type, attrs,
3122 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
3123 if (ret)
3124 return ret;
3125 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
3126 if (IS_ERR(action))
3127 return PTR_ERR(action);
3128
3129 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
3130 IB_FLOW_ACTION_UNSPECIFIED);
3131
3132 return 0;
3133 }
3134
mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev * ibdev,u8 packet_reformat_type,u8 ft_type)3135 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
3136 u8 packet_reformat_type,
3137 u8 ft_type)
3138 {
3139 switch (packet_reformat_type) {
3140 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3141 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3142 return MLX5_CAP_FLOWTABLE(ibdev->mdev,
3143 encap_general_header);
3144 break;
3145 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3146 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3147 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
3148 reformat_l2_to_l3_tunnel);
3149 break;
3150 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3151 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3152 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
3153 reformat_l3_tunnel_to_l2);
3154 break;
3155 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
3156 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3157 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
3158 break;
3159 default:
3160 break;
3161 }
3162
3163 return false;
3164 }
3165
mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt,u8 * prm_prt)3166 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
3167 {
3168 switch (dv_prt) {
3169 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3170 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
3171 break;
3172 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3173 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
3174 break;
3175 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3176 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3177 break;
3178 default:
3179 return -EINVAL;
3180 }
3181
3182 return 0;
3183 }
3184
mlx5_ib_flow_action_create_packet_reformat_ctx(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_action * maction,u8 ft_type,u8 dv_prt,void * in,size_t len)3185 static int mlx5_ib_flow_action_create_packet_reformat_ctx(
3186 struct mlx5_ib_dev *dev,
3187 struct mlx5_ib_flow_action *maction,
3188 u8 ft_type, u8 dv_prt,
3189 void *in, size_t len)
3190 {
3191 struct mlx5_pkt_reformat_params reformat_params;
3192 enum mlx5_flow_namespace_type namespace;
3193 u8 prm_prt;
3194 int ret;
3195
3196 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3197 if (ret)
3198 return ret;
3199
3200 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
3201 if (ret)
3202 return ret;
3203
3204 memset(&reformat_params, 0, sizeof(reformat_params));
3205 reformat_params.type = prm_prt;
3206 reformat_params.size = len;
3207 reformat_params.data = in;
3208 maction->flow_action_raw.pkt_reformat =
3209 mlx5_packet_reformat_alloc(dev->mdev, &reformat_params,
3210 namespace);
3211 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
3212 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
3213 return ret;
3214 }
3215
3216 maction->flow_action_raw.sub_type =
3217 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
3218 maction->flow_action_raw.dev = dev;
3219
3220 return 0;
3221 }
3222
UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)3223 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
3224 struct uverbs_attr_bundle *attrs)
3225 {
3226 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
3227 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
3228 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3229 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
3230 enum mlx5_ib_uapi_flow_table_type ft_type;
3231 struct mlx5_ib_flow_action *maction;
3232 int ret;
3233
3234 ret = uverbs_get_const(&ft_type, attrs,
3235 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
3236 if (ret)
3237 return ret;
3238
3239 ret = uverbs_get_const(&dv_prt, attrs,
3240 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
3241 if (ret)
3242 return ret;
3243
3244 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
3245 return -EOPNOTSUPP;
3246
3247 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3248 if (!maction)
3249 return -ENOMEM;
3250
3251 if (dv_prt ==
3252 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
3253 maction->flow_action_raw.sub_type =
3254 MLX5_IB_FLOW_ACTION_DECAP;
3255 maction->flow_action_raw.dev = mdev;
3256 } else {
3257 void *in;
3258 int len;
3259
3260 in = uverbs_attr_get_alloced_ptr(attrs,
3261 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3262 if (IS_ERR(in)) {
3263 ret = PTR_ERR(in);
3264 goto free_maction;
3265 }
3266
3267 len = uverbs_attr_get_len(attrs,
3268 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3269
3270 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
3271 maction, ft_type, dv_prt, in, len);
3272 if (ret)
3273 goto free_maction;
3274 }
3275
3276 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
3277 IB_FLOW_ACTION_UNSPECIFIED);
3278 return 0;
3279
3280 free_maction:
3281 kfree(maction);
3282 return ret;
3283 }
3284
3285 DECLARE_UVERBS_NAMED_METHOD(
3286 MLX5_IB_METHOD_CREATE_FLOW,
3287 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3288 UVERBS_OBJECT_FLOW,
3289 UVERBS_ACCESS_NEW,
3290 UA_MANDATORY),
3291 UVERBS_ATTR_PTR_IN(
3292 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
3293 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3294 UA_MANDATORY,
3295 UA_ALLOC_AND_COPY),
3296 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
3297 MLX5_IB_OBJECT_FLOW_MATCHER,
3298 UVERBS_ACCESS_READ,
3299 UA_MANDATORY),
3300 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
3301 UVERBS_OBJECT_QP,
3302 UVERBS_ACCESS_READ),
3303 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
3304 MLX5_IB_OBJECT_DEVX_OBJ,
3305 UVERBS_ACCESS_READ),
3306 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
3307 UVERBS_OBJECT_FLOW_ACTION,
3308 UVERBS_ACCESS_READ, 1,
3309 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
3310 UA_OPTIONAL),
3311 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
3312 UVERBS_ATTR_TYPE(u32),
3313 UA_OPTIONAL),
3314 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
3315 MLX5_IB_OBJECT_DEVX_OBJ,
3316 UVERBS_ACCESS_READ, 1, 1,
3317 UA_OPTIONAL),
3318 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
3319 UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
3320 UA_OPTIONAL,
3321 UA_ALLOC_AND_COPY),
3322 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
3323 enum mlx5_ib_create_flow_flags,
3324 UA_OPTIONAL));
3325
3326 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3327 MLX5_IB_METHOD_DESTROY_FLOW,
3328 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3329 UVERBS_OBJECT_FLOW,
3330 UVERBS_ACCESS_DESTROY,
3331 UA_MANDATORY));
3332
3333 ADD_UVERBS_METHODS(mlx5_ib_fs,
3334 UVERBS_OBJECT_FLOW,
3335 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
3336 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
3337
3338 DECLARE_UVERBS_NAMED_METHOD(
3339 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
3340 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
3341 UVERBS_OBJECT_FLOW_ACTION,
3342 UVERBS_ACCESS_NEW,
3343 UA_MANDATORY),
3344 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3345 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
3346 set_add_copy_action_in_auto)),
3347 UA_MANDATORY,
3348 UA_ALLOC_AND_COPY),
3349 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
3350 enum mlx5_ib_uapi_flow_table_type,
3351 UA_MANDATORY));
3352
3353 DECLARE_UVERBS_NAMED_METHOD(
3354 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
3355 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
3356 UVERBS_OBJECT_FLOW_ACTION,
3357 UVERBS_ACCESS_NEW,
3358 UA_MANDATORY),
3359 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
3360 UVERBS_ATTR_MIN_SIZE(1),
3361 UA_ALLOC_AND_COPY,
3362 UA_OPTIONAL),
3363 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
3364 enum mlx5_ib_uapi_flow_action_packet_reformat_type,
3365 UA_MANDATORY),
3366 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
3367 enum mlx5_ib_uapi_flow_table_type,
3368 UA_MANDATORY));
3369
3370 ADD_UVERBS_METHODS(
3371 mlx5_ib_flow_actions,
3372 UVERBS_OBJECT_FLOW_ACTION,
3373 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
3374 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
3375
3376 DECLARE_UVERBS_NAMED_METHOD(
3377 MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
3378 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
3379 MLX5_IB_OBJECT_FLOW_MATCHER,
3380 UVERBS_ACCESS_NEW,
3381 UA_MANDATORY),
3382 UVERBS_ATTR_PTR_IN(
3383 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
3384 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3385 UA_MANDATORY),
3386 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
3387 mlx5_ib_flow_type,
3388 UA_MANDATORY),
3389 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
3390 UVERBS_ATTR_TYPE(u8),
3391 UA_MANDATORY),
3392 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
3393 enum ib_flow_flags,
3394 UA_OPTIONAL),
3395 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
3396 enum mlx5_ib_uapi_flow_table_type,
3397 UA_OPTIONAL),
3398 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,
3399 UVERBS_ATTR_TYPE(u32),
3400 UA_OPTIONAL));
3401
3402 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3403 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
3404 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
3405 MLX5_IB_OBJECT_FLOW_MATCHER,
3406 UVERBS_ACCESS_DESTROY,
3407 UA_MANDATORY));
3408
3409 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
3410 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
3411 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
3412 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
3413
3414 DECLARE_UVERBS_NAMED_METHOD(
3415 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,
3416 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,
3417 MLX5_IB_OBJECT_STEERING_ANCHOR,
3418 UVERBS_ACCESS_NEW,
3419 UA_MANDATORY),
3420 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,
3421 enum mlx5_ib_uapi_flow_table_type,
3422 UA_MANDATORY),
3423 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,
3424 UVERBS_ATTR_TYPE(u16),
3425 UA_MANDATORY),
3426 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3427 UVERBS_ATTR_TYPE(u32),
3428 UA_MANDATORY));
3429
3430 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3431 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,
3432 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,
3433 MLX5_IB_OBJECT_STEERING_ANCHOR,
3434 UVERBS_ACCESS_DESTROY,
3435 UA_MANDATORY));
3436
3437 DECLARE_UVERBS_NAMED_OBJECT(
3438 MLX5_IB_OBJECT_STEERING_ANCHOR,
3439 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),
3440 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),
3441 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));
3442
3443 const struct uapi_definition mlx5_ib_flow_defs[] = {
3444 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3445 MLX5_IB_OBJECT_FLOW_MATCHER),
3446 UAPI_DEF_CHAIN_OBJ_TREE(
3447 UVERBS_OBJECT_FLOW,
3448 &mlx5_ib_fs),
3449 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3450 &mlx5_ib_flow_actions),
3451 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3452 MLX5_IB_OBJECT_STEERING_ANCHOR,
3453 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),
3454 {},
3455 };
3456
3457 static const struct ib_device_ops flow_ops = {
3458 .create_flow = mlx5_ib_create_flow,
3459 .destroy_flow = mlx5_ib_destroy_flow,
3460 .destroy_flow_action = mlx5_ib_destroy_flow_action,
3461 };
3462
mlx5_ib_fs_init(struct mlx5_ib_dev * dev)3463 int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
3464 {
3465 int i, j;
3466
3467 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
3468
3469 if (!dev->flow_db)
3470 return -ENOMEM;
3471
3472 for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {
3473 dev->flow_db->rdma_transport_rx[i] =
3474 kcalloc(dev->num_ports,
3475 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3476 if (!dev->flow_db->rdma_transport_rx[i])
3477 goto free_rdma_transport_rx;
3478 }
3479
3480 for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {
3481 dev->flow_db->rdma_transport_tx[j] =
3482 kcalloc(dev->num_ports,
3483 sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3484 if (!dev->flow_db->rdma_transport_tx[j])
3485 goto free_rdma_transport_tx;
3486 }
3487
3488 mutex_init(&dev->flow_db->lock);
3489
3490 ib_set_device_ops(&dev->ib_dev, &flow_ops);
3491 return 0;
3492
3493 free_rdma_transport_tx:
3494 while (j--)
3495 kfree(dev->flow_db->rdma_transport_tx[j]);
3496 free_rdma_transport_rx:
3497 while (i--)
3498 kfree(dev->flow_db->rdma_transport_rx[i]);
3499 kfree(dev->flow_db);
3500 return -ENOMEM;
3501 }
3502