1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.
3
4 #include "rss.h"
5
6 #define mlx5e_rss_warn(__dev, format, ...) \
7 dev_warn((__dev)->device, "%s:%d:(pid %d): " format, \
8 __func__, __LINE__, current->pid, \
9 ##__VA_ARGS__)
10
11 static const struct mlx5e_rss_params_traffic_type rss_default_config[MLX5E_NUM_INDIR_TIRS] = {
12 [MLX5_TT_IPV4_TCP] = {
13 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
14 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
15 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
16 },
17 [MLX5_TT_IPV6_TCP] = {
18 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
19 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
20 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
21 },
22 [MLX5_TT_IPV4_UDP] = {
23 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
24 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
25 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
26 },
27 [MLX5_TT_IPV6_UDP] = {
28 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
29 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
30 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
31 },
32 [MLX5_TT_IPV4_IPSEC_AH] = {
33 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
34 .l4_prot_type = 0,
35 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
36 },
37 [MLX5_TT_IPV6_IPSEC_AH] = {
38 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
39 .l4_prot_type = 0,
40 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
41 },
42 [MLX5_TT_IPV4_IPSEC_ESP] = {
43 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
44 .l4_prot_type = 0,
45 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
46 },
47 [MLX5_TT_IPV6_IPSEC_ESP] = {
48 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
49 .l4_prot_type = 0,
50 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
51 },
52 [MLX5_TT_IPV4] = {
53 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
54 .l4_prot_type = 0,
55 .rx_hash_fields = MLX5_HASH_IP,
56 },
57 [MLX5_TT_IPV6] = {
58 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
59 .l4_prot_type = 0,
60 .rx_hash_fields = MLX5_HASH_IP,
61 },
62 };
63
64 struct mlx5e_rss_params_traffic_type
mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt)65 mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt)
66 {
67 return rss_default_config[tt];
68 }
69
70 struct mlx5e_rss {
71 struct mlx5e_rss_params_hash hash;
72 struct mlx5e_rss_params_indir indir;
73 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
74 struct mlx5e_tir *tir[MLX5E_NUM_INDIR_TIRS];
75 struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS];
76 struct mlx5e_rqt rqt;
77 struct mlx5_core_dev *mdev;
78 u32 drop_rqn;
79 bool inner_ft_support;
80 bool enabled;
81 refcount_t refcnt;
82 };
83
mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss * rss,u32 num_channels)84 void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels)
85 {
86 rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels);
87 }
88
mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir * indir,struct mlx5_core_dev * mdev,u32 actual_table_size,u32 max_table_size)89 int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev,
90 u32 actual_table_size, u32 max_table_size)
91 {
92 indir->table = kvmalloc_array(max_table_size, sizeof(*indir->table), GFP_KERNEL);
93 if (!indir->table)
94 return -ENOMEM;
95
96 indir->max_table_size = max_table_size;
97 indir->actual_table_size = actual_table_size;
98
99 return 0;
100 }
101
mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir * indir)102 void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir)
103 {
104 kvfree(indir->table);
105 }
106
mlx5e_rss_copy(struct mlx5e_rss * to,const struct mlx5e_rss * from)107 static int mlx5e_rss_copy(struct mlx5e_rss *to, const struct mlx5e_rss *from)
108 {
109 u32 *dst_indir_table;
110
111 if (to->indir.actual_table_size != from->indir.actual_table_size ||
112 to->indir.max_table_size != from->indir.max_table_size) {
113 mlx5e_rss_warn(to->mdev,
114 "Failed to copy RSS due to size mismatch, src (actual %u, max %u) != dst (actual %u, max %u)\n",
115 from->indir.actual_table_size, from->indir.max_table_size,
116 to->indir.actual_table_size, to->indir.max_table_size);
117 return -EINVAL;
118 }
119
120 dst_indir_table = to->indir.table;
121 *to = *from;
122 to->indir.table = dst_indir_table;
123 memcpy(to->indir.table, from->indir.table,
124 from->indir.actual_table_size * sizeof(*from->indir.table));
125 return 0;
126 }
127
mlx5e_rss_init_copy(const struct mlx5e_rss * from)128 static struct mlx5e_rss *mlx5e_rss_init_copy(const struct mlx5e_rss *from)
129 {
130 struct mlx5e_rss *rss;
131 int err;
132
133 rss = kvzalloc(sizeof(*rss), GFP_KERNEL);
134 if (!rss)
135 return ERR_PTR(-ENOMEM);
136
137 err = mlx5e_rss_params_indir_init(&rss->indir, from->mdev, from->indir.actual_table_size,
138 from->indir.max_table_size);
139 if (err)
140 goto err_free_rss;
141
142 err = mlx5e_rss_copy(rss, from);
143 if (err)
144 goto err_free_indir;
145
146 return rss;
147
148 err_free_indir:
149 mlx5e_rss_params_indir_cleanup(&rss->indir);
150 err_free_rss:
151 kvfree(rss);
152 return ERR_PTR(err);
153 }
154
mlx5e_rss_params_init(struct mlx5e_rss * rss)155 static void mlx5e_rss_params_init(struct mlx5e_rss *rss)
156 {
157 enum mlx5_traffic_types tt;
158
159 rss->hash.hfunc = ETH_RSS_HASH_TOP;
160 netdev_rss_key_fill(rss->hash.toeplitz_hash_key,
161 sizeof(rss->hash.toeplitz_hash_key));
162 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
163 rss->rx_hash_fields[tt] =
164 mlx5e_rss_get_default_tt_config(tt).rx_hash_fields;
165 }
166
rss_get_tirp(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)167 static struct mlx5e_tir **rss_get_tirp(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
168 bool inner)
169 {
170 return inner ? &rss->inner_tir[tt] : &rss->tir[tt];
171 }
172
rss_get_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)173 static struct mlx5e_tir *rss_get_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
174 bool inner)
175 {
176 return *rss_get_tirp(rss, tt, inner);
177 }
178
179 static struct mlx5e_rss_params_traffic_type
mlx5e_rss_get_tt_config(struct mlx5e_rss * rss,enum mlx5_traffic_types tt)180 mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
181 {
182 struct mlx5e_rss_params_traffic_type rss_tt;
183
184 rss_tt = mlx5e_rss_get_default_tt_config(tt);
185 rss_tt.rx_hash_fields = rss->rx_hash_fields[tt];
186 return rss_tt;
187 }
188
mlx5e_rss_create_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,const struct mlx5e_packet_merge_param * init_pkt_merge_param,bool inner)189 static int mlx5e_rss_create_tir(struct mlx5e_rss *rss,
190 enum mlx5_traffic_types tt,
191 const struct mlx5e_packet_merge_param *init_pkt_merge_param,
192 bool inner)
193 {
194 struct mlx5e_rss_params_traffic_type rss_tt;
195 struct mlx5e_tir_builder *builder;
196 struct mlx5e_tir **tir_p;
197 struct mlx5e_tir *tir;
198 u32 rqtn;
199 int err;
200
201 if (inner && !rss->inner_ft_support) {
202 mlx5e_rss_warn(rss->mdev,
203 "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n",
204 tt);
205 return -EINVAL;
206 }
207
208 tir_p = rss_get_tirp(rss, tt, inner);
209 if (*tir_p)
210 return -EINVAL;
211
212 tir = kvzalloc(sizeof(*tir), GFP_KERNEL);
213 if (!tir)
214 return -ENOMEM;
215
216 builder = mlx5e_tir_builder_alloc(false);
217 if (!builder) {
218 err = -ENOMEM;
219 goto free_tir;
220 }
221
222 rqtn = mlx5e_rqt_get_rqtn(&rss->rqt);
223 mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn,
224 rqtn, rss->inner_ft_support);
225 mlx5e_tir_builder_build_packet_merge(builder, init_pkt_merge_param);
226 rss_tt = mlx5e_rss_get_tt_config(rss, tt);
227 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
228
229 err = mlx5e_tir_init(tir, builder, rss->mdev, true);
230 mlx5e_tir_builder_free(builder);
231 if (err) {
232 mlx5e_rss_warn(rss->mdev, "Failed to create %sindirect TIR: err = %d, tt = %d\n",
233 inner ? "inner " : "", err, tt);
234 goto free_tir;
235 }
236
237 *tir_p = tir;
238 return 0;
239
240 free_tir:
241 kvfree(tir);
242 return err;
243 }
244
mlx5e_rss_destroy_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)245 static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
246 bool inner)
247 {
248 struct mlx5e_tir **tir_p;
249 struct mlx5e_tir *tir;
250
251 tir_p = rss_get_tirp(rss, tt, inner);
252 if (!*tir_p)
253 return;
254
255 tir = *tir_p;
256 mlx5e_tir_destroy(tir);
257 kvfree(tir);
258 *tir_p = NULL;
259 }
260
mlx5e_rss_create_tirs(struct mlx5e_rss * rss,const struct mlx5e_packet_merge_param * init_pkt_merge_param,bool inner)261 static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss,
262 const struct mlx5e_packet_merge_param *init_pkt_merge_param,
263 bool inner)
264 {
265 enum mlx5_traffic_types tt, max_tt;
266 int err;
267
268 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
269 err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
270 if (err)
271 goto err_destroy_tirs;
272 }
273
274 return 0;
275
276 err_destroy_tirs:
277 max_tt = tt;
278 for (tt = 0; tt < max_tt; tt++)
279 mlx5e_rss_destroy_tir(rss, tt, inner);
280 return err;
281 }
282
mlx5e_rss_destroy_tirs(struct mlx5e_rss * rss,bool inner)283 static void mlx5e_rss_destroy_tirs(struct mlx5e_rss *rss, bool inner)
284 {
285 enum mlx5_traffic_types tt;
286
287 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
288 mlx5e_rss_destroy_tir(rss, tt, inner);
289 }
290
mlx5e_rss_update_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)291 static int mlx5e_rss_update_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
292 bool inner)
293 {
294 struct mlx5e_rss_params_traffic_type rss_tt;
295 struct mlx5e_tir_builder *builder;
296 struct mlx5e_tir *tir;
297 int err;
298
299 tir = rss_get_tir(rss, tt, inner);
300 if (!tir)
301 return 0;
302
303 builder = mlx5e_tir_builder_alloc(true);
304 if (!builder)
305 return -ENOMEM;
306
307 rss_tt = mlx5e_rss_get_tt_config(rss, tt);
308
309 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
310 err = mlx5e_tir_modify(tir, builder);
311
312 mlx5e_tir_builder_free(builder);
313 return err;
314 }
315
mlx5e_rss_update_tirs(struct mlx5e_rss * rss)316 static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss)
317 {
318 enum mlx5_traffic_types tt;
319 int err, retval;
320
321 retval = 0;
322
323 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
324 err = mlx5e_rss_update_tir(rss, tt, false);
325 if (err) {
326 retval = retval ? : err;
327 mlx5e_rss_warn(rss->mdev,
328 "Failed to update RSS hash of indirect TIR for traffic type %d: err = %d\n",
329 tt, err);
330 }
331
332 if (!rss->inner_ft_support)
333 continue;
334
335 err = mlx5e_rss_update_tir(rss, tt, true);
336 if (err) {
337 retval = retval ? : err;
338 mlx5e_rss_warn(rss->mdev,
339 "Failed to update RSS hash of inner indirect TIR for traffic type %d: err = %d\n",
340 tt, err);
341 }
342 }
343 return retval;
344 }
345
mlx5e_rss_init_no_tirs(struct mlx5e_rss * rss)346 static int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss)
347 {
348 mlx5e_rss_params_init(rss);
349 refcount_set(&rss->refcnt, 1);
350
351 return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true,
352 rss->drop_rqn, rss->indir.max_table_size);
353 }
354
mlx5e_rss_init(struct mlx5_core_dev * mdev,bool inner_ft_support,u32 drop_rqn,const struct mlx5e_packet_merge_param * init_pkt_merge_param,enum mlx5e_rss_init_type type,unsigned int nch,unsigned int max_nch)355 struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn,
356 const struct mlx5e_packet_merge_param *init_pkt_merge_param,
357 enum mlx5e_rss_init_type type, unsigned int nch,
358 unsigned int max_nch)
359 {
360 struct mlx5e_rss *rss;
361 int err;
362
363 rss = kvzalloc(sizeof(*rss), GFP_KERNEL);
364 if (!rss)
365 return ERR_PTR(-ENOMEM);
366
367 err = mlx5e_rss_params_indir_init(&rss->indir, mdev,
368 mlx5e_rqt_size(mdev, nch),
369 mlx5e_rqt_size(mdev, max_nch));
370 if (err)
371 goto err_free_rss;
372
373 rss->mdev = mdev;
374 rss->inner_ft_support = inner_ft_support;
375 rss->drop_rqn = drop_rqn;
376
377 err = mlx5e_rss_init_no_tirs(rss);
378 if (err)
379 goto err_free_indir;
380
381 if (type == MLX5E_RSS_INIT_NO_TIRS)
382 goto out;
383
384 err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, false);
385 if (err)
386 goto err_destroy_rqt;
387
388 if (inner_ft_support) {
389 err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, true);
390 if (err)
391 goto err_destroy_tirs;
392 }
393
394 out:
395 return rss;
396
397 err_destroy_tirs:
398 mlx5e_rss_destroy_tirs(rss, false);
399 err_destroy_rqt:
400 mlx5e_rqt_destroy(&rss->rqt);
401 err_free_indir:
402 mlx5e_rss_params_indir_cleanup(&rss->indir);
403 err_free_rss:
404 kvfree(rss);
405 return ERR_PTR(err);
406 }
407
mlx5e_rss_cleanup(struct mlx5e_rss * rss)408 int mlx5e_rss_cleanup(struct mlx5e_rss *rss)
409 {
410 if (!refcount_dec_if_one(&rss->refcnt))
411 return -EBUSY;
412
413 mlx5e_rss_destroy_tirs(rss, false);
414
415 if (rss->inner_ft_support)
416 mlx5e_rss_destroy_tirs(rss, true);
417
418 mlx5e_rqt_destroy(&rss->rqt);
419 mlx5e_rss_params_indir_cleanup(&rss->indir);
420 kvfree(rss);
421
422 return 0;
423 }
424
mlx5e_rss_refcnt_inc(struct mlx5e_rss * rss)425 void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss)
426 {
427 refcount_inc(&rss->refcnt);
428 }
429
mlx5e_rss_refcnt_dec(struct mlx5e_rss * rss)430 void mlx5e_rss_refcnt_dec(struct mlx5e_rss *rss)
431 {
432 refcount_dec(&rss->refcnt);
433 }
434
mlx5e_rss_refcnt_read(struct mlx5e_rss * rss)435 unsigned int mlx5e_rss_refcnt_read(struct mlx5e_rss *rss)
436 {
437 return refcount_read(&rss->refcnt);
438 }
439
mlx5e_rss_get_tirn(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)440 u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
441 bool inner)
442 {
443 struct mlx5e_tir *tir;
444
445 WARN_ON(inner && !rss->inner_ft_support);
446 tir = rss_get_tir(rss, tt, inner);
447 WARN_ON(!tir);
448
449 return mlx5e_tir_get_tirn(tir);
450 }
451
452 /* Fill the "tirn" output parameter.
453 * Create the requested TIR if it's its first usage.
454 */
mlx5e_rss_obtain_tirn(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,const struct mlx5e_packet_merge_param * init_pkt_merge_param,bool inner,u32 * tirn)455 int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss,
456 enum mlx5_traffic_types tt,
457 const struct mlx5e_packet_merge_param *init_pkt_merge_param,
458 bool inner, u32 *tirn)
459 {
460 struct mlx5e_tir *tir;
461
462 tir = rss_get_tir(rss, tt, inner);
463 if (!tir) { /* TIR doesn't exist, create one */
464 int err;
465
466 err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner);
467 if (err)
468 return err;
469 tir = rss_get_tir(rss, tt, inner);
470 }
471
472 *tirn = mlx5e_tir_get_tirn(tir);
473 return 0;
474 }
475
mlx5e_rss_apply(struct mlx5e_rss * rss,u32 * rqns,unsigned int num_rqns)476 static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns)
477 {
478 int err;
479
480 err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, num_rqns, rss->hash.hfunc, &rss->indir);
481 if (err)
482 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n",
483 mlx5e_rqt_get_rqtn(&rss->rqt), err);
484 return err;
485 }
486
mlx5e_rss_enable(struct mlx5e_rss * rss,u32 * rqns,unsigned int num_rqns)487 void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns)
488 {
489 rss->enabled = true;
490 mlx5e_rss_apply(rss, rqns, num_rqns);
491 }
492
mlx5e_rss_disable(struct mlx5e_rss * rss)493 void mlx5e_rss_disable(struct mlx5e_rss *rss)
494 {
495 int err;
496
497 rss->enabled = false;
498 err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn);
499 if (err)
500 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n",
501 mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err);
502 }
503
mlx5e_rss_packet_merge_set_param(struct mlx5e_rss * rss,struct mlx5e_packet_merge_param * pkt_merge_param)504 int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss,
505 struct mlx5e_packet_merge_param *pkt_merge_param)
506 {
507 struct mlx5e_tir_builder *builder;
508 enum mlx5_traffic_types tt;
509 int err, final_err;
510
511 builder = mlx5e_tir_builder_alloc(true);
512 if (!builder)
513 return -ENOMEM;
514
515 mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
516
517 final_err = 0;
518
519 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
520 struct mlx5e_tir *tir;
521
522 tir = rss_get_tir(rss, tt, false);
523 if (!tir)
524 goto inner_tir;
525 err = mlx5e_tir_modify(tir, builder);
526 if (err) {
527 mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of indirect TIR %#x for traffic type %d: err = %d\n",
528 mlx5e_tir_get_tirn(tir), tt, err);
529 if (!final_err)
530 final_err = err;
531 }
532
533 inner_tir:
534 if (!rss->inner_ft_support)
535 continue;
536
537 tir = rss_get_tir(rss, tt, true);
538 if (!tir)
539 continue;
540 err = mlx5e_tir_modify(tir, builder);
541 if (err) {
542 mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of inner indirect TIR %#x for traffic type %d: err = %d\n",
543 mlx5e_tir_get_tirn(tir), tt, err);
544 if (!final_err)
545 final_err = err;
546 }
547 }
548
549 mlx5e_tir_builder_free(builder);
550 return final_err;
551 }
552
mlx5e_rss_get_rxfh(struct mlx5e_rss * rss,u32 * indir,u8 * key,u8 * hfunc)553 int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc)
554 {
555 if (indir)
556 memcpy(indir, rss->indir.table,
557 rss->indir.actual_table_size * sizeof(*rss->indir.table));
558
559 if (key)
560 memcpy(key, rss->hash.toeplitz_hash_key,
561 sizeof(rss->hash.toeplitz_hash_key));
562
563 if (hfunc)
564 *hfunc = rss->hash.hfunc;
565
566 return 0;
567 }
568
mlx5e_rss_set_rxfh(struct mlx5e_rss * rss,const u32 * indir,const u8 * key,const u8 * hfunc,u32 * rqns,unsigned int num_rqns)569 int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir,
570 const u8 *key, const u8 *hfunc,
571 u32 *rqns, unsigned int num_rqns)
572 {
573 bool changed_indir = false;
574 bool changed_hash = false;
575 struct mlx5e_rss *old_rss;
576 int err = 0;
577
578 old_rss = mlx5e_rss_init_copy(rss);
579 if (IS_ERR(old_rss))
580 return PTR_ERR(old_rss);
581
582 if (hfunc && *hfunc != rss->hash.hfunc) {
583 switch (*hfunc) {
584 case ETH_RSS_HASH_XOR:
585 case ETH_RSS_HASH_TOP:
586 break;
587 default:
588 err = -EINVAL;
589 goto out;
590 }
591 changed_hash = true;
592 changed_indir = true;
593 rss->hash.hfunc = *hfunc;
594 }
595
596 if (key) {
597 if (rss->hash.hfunc == ETH_RSS_HASH_TOP)
598 changed_hash = true;
599 memcpy(rss->hash.toeplitz_hash_key, key,
600 sizeof(rss->hash.toeplitz_hash_key));
601 }
602
603 if (indir) {
604 changed_indir = true;
605
606 memcpy(rss->indir.table, indir,
607 rss->indir.actual_table_size * sizeof(*rss->indir.table));
608 }
609
610 if (changed_indir && rss->enabled) {
611 err = mlx5e_rss_apply(rss, rqns, num_rqns);
612 if (err) {
613 mlx5e_rss_copy(rss, old_rss);
614 goto out;
615 }
616 }
617
618 if (changed_hash)
619 mlx5e_rss_update_tirs(rss);
620
621 out:
622 mlx5e_rss_params_indir_cleanup(&old_rss->indir);
623 kvfree(old_rss);
624
625 return err;
626 }
627
mlx5e_rss_get_hash(struct mlx5e_rss * rss)628 struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss)
629 {
630 return rss->hash;
631 }
632
mlx5e_rss_get_hash_fields(struct mlx5e_rss * rss,enum mlx5_traffic_types tt)633 u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
634 {
635 return rss->rx_hash_fields[tt];
636 }
637
mlx5e_rss_set_hash_fields(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,u8 rx_hash_fields)638 int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
639 u8 rx_hash_fields)
640 {
641 u8 old_rx_hash_fields;
642 int err;
643
644 old_rx_hash_fields = rss->rx_hash_fields[tt];
645
646 if (old_rx_hash_fields == rx_hash_fields)
647 return 0;
648
649 rss->rx_hash_fields[tt] = rx_hash_fields;
650
651 err = mlx5e_rss_update_tir(rss, tt, false);
652 if (err) {
653 rss->rx_hash_fields[tt] = old_rx_hash_fields;
654 mlx5e_rss_warn(rss->mdev,
655 "Failed to update RSS hash fields of indirect TIR for traffic type %d: err = %d\n",
656 tt, err);
657 return err;
658 }
659
660 if (!(rss->inner_ft_support))
661 return 0;
662
663 err = mlx5e_rss_update_tir(rss, tt, true);
664 if (err) {
665 /* Partial update happened. Try to revert - it may fail too, but
666 * there is nothing more we can do.
667 */
668 rss->rx_hash_fields[tt] = old_rx_hash_fields;
669 mlx5e_rss_warn(rss->mdev,
670 "Failed to update RSS hash fields of inner indirect TIR for traffic type %d: err = %d\n",
671 tt, err);
672 if (mlx5e_rss_update_tir(rss, tt, false))
673 mlx5e_rss_warn(rss->mdev,
674 "Partial update of RSS hash fields happened: failed to revert indirect TIR for traffic type %d to the old values\n",
675 tt);
676 }
677
678 return err;
679 }
680
mlx5e_rss_set_indir_uniform(struct mlx5e_rss * rss,unsigned int nch)681 void mlx5e_rss_set_indir_uniform(struct mlx5e_rss *rss, unsigned int nch)
682 {
683 mlx5e_rss_params_indir_init_uniform(&rss->indir, nch);
684 }
685