1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3
4 #include "rqt.h"
5 #include <linux/mlx5/transobj.h>
6
mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir * indir,unsigned int num_channels)7 void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir,
8 unsigned int num_channels)
9 {
10 unsigned int i;
11
12 for (i = 0; i < indir->actual_table_size; i++)
13 indir->table[i] = i % num_channels;
14 }
15
mlx5e_rqt_init(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,u16 max_size,u32 * init_rqns,u16 init_size)16 static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
17 u16 max_size, u32 *init_rqns, u16 init_size)
18 {
19 void *rqtc;
20 int inlen;
21 int err;
22 u32 *in;
23 int i;
24
25 rqt->mdev = mdev;
26 rqt->size = max_size;
27
28 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * init_size;
29 in = kvzalloc(inlen, GFP_KERNEL);
30 if (!in)
31 return -ENOMEM;
32
33 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
34
35 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size);
36
37 MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size);
38 for (i = 0; i < init_size; i++)
39 MLX5_SET(rqtc, rqtc, rq_num[i], init_rqns[i]);
40
41 err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn);
42
43 kvfree(in);
44 return err;
45 }
46
mlx5e_rqt_init_direct(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,bool indir_enabled,u32 init_rqn,u32 indir_table_size)47 int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
48 bool indir_enabled, u32 init_rqn, u32 indir_table_size)
49 {
50 u16 max_size = indir_enabled ? indir_table_size : 1;
51
52 return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, 1);
53 }
54
mlx5e_bits_invert(unsigned long a,int size)55 static int mlx5e_bits_invert(unsigned long a, int size)
56 {
57 int inv = 0;
58 int i;
59
60 for (i = 0; i < size; i++)
61 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
62
63 return inv;
64 }
65
mlx5e_calc_indir_rqns(u32 * rss_rqns,u32 * rqns,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)66 static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, unsigned int num_rqns,
67 u8 hfunc, struct mlx5e_rss_params_indir *indir)
68 {
69 unsigned int i;
70
71 for (i = 0; i < indir->actual_table_size; i++) {
72 unsigned int ix = i;
73
74 if (hfunc == ETH_RSS_HASH_XOR)
75 ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size));
76
77 ix = indir->table[ix];
78
79 if (WARN_ON(ix >= num_rqns))
80 /* Could be a bug in the driver or in the kernel part of
81 * ethtool: indir table refers to non-existent RQs.
82 */
83 return -EINVAL;
84 rss_rqns[i] = rqns[ix];
85 }
86
87 return 0;
88 }
89
mlx5e_rqt_init_indir(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,u32 * rqns,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)90 int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
91 u32 *rqns, unsigned int num_rqns,
92 u8 hfunc, struct mlx5e_rss_params_indir *indir)
93 {
94 u32 *rss_rqns;
95 int err;
96
97 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
98 if (!rss_rqns)
99 return -ENOMEM;
100
101 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir);
102 if (err)
103 goto out;
104
105 err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns,
106 indir->actual_table_size);
107
108 out:
109 kvfree(rss_rqns);
110 return err;
111 }
112
113 #define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2
114
mlx5e_rqt_size(struct mlx5_core_dev * mdev,unsigned int num_channels)115 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
116 {
117 u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,
118 roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR));
119 u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size);
120
121 return min_t(u32, rqt_size, max_cap_rqt_size);
122 }
123
mlx5e_rqt_destroy(struct mlx5e_rqt * rqt)124 void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)
125 {
126 mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn);
127 }
128
mlx5e_rqt_redirect(struct mlx5e_rqt * rqt,u32 * rqns,unsigned int size)129 static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int size)
130 {
131 unsigned int i;
132 void *rqtc;
133 int inlen;
134 u32 *in;
135 int err;
136
137 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * size;
138 in = kvzalloc(inlen, GFP_KERNEL);
139 if (!in)
140 return -ENOMEM;
141
142 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
143
144 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
145 MLX5_SET(rqtc, rqtc, rqt_actual_size, size);
146 for (i = 0; i < size; i++)
147 MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]);
148
149 err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen);
150
151 kvfree(in);
152 return err;
153 }
154
mlx5e_rqt_redirect_direct(struct mlx5e_rqt * rqt,u32 rqn)155 int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn)
156 {
157 return mlx5e_rqt_redirect(rqt, &rqn, 1);
158 }
159
mlx5e_rqt_redirect_indir(struct mlx5e_rqt * rqt,u32 * rqns,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)160 int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns,
161 u8 hfunc, struct mlx5e_rss_params_indir *indir)
162 {
163 u32 *rss_rqns;
164 int err;
165
166 if (WARN_ON(rqt->size != indir->max_table_size))
167 return -EINVAL;
168
169 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
170 if (!rss_rqns)
171 return -ENOMEM;
172
173 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir);
174 if (err)
175 goto out;
176
177 err = mlx5e_rqt_redirect(rqt, rss_rqns, indir->actual_table_size);
178
179 out:
180 kvfree(rss_rqns);
181 return err;
182 }
183