1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2005-2018 Solarflare Communications Inc.
5 * Copyright 2019-2020 Xilinx Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation, incorporated herein by reference.
10 */
11
12 #include "mcdi_filters.h"
13 #include "mcdi.h"
14 #include "nic.h"
15 #include "rx_common.h"
16
17 /* The maximum size of a shared RSS context */
18 /* TODO: this should really be from the mcdi protocol export */
19 #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL
20
21 #define EFX_EF10_FILTER_ID_INVALID 0xffff
22
23 /* An arbitrary search limit for the software hash table */
24 #define EFX_EF10_FILTER_SEARCH_LIMIT 200
25
26 static struct efx_filter_spec *
efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table * table,unsigned int filter_idx)27 efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table,
28 unsigned int filter_idx)
29 {
30 return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
31 ~EFX_EF10_FILTER_FLAGS);
32 }
33
34 static unsigned int
efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table * table,unsigned int filter_idx)35 efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table,
36 unsigned int filter_idx)
37 {
38 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
39 }
40
efx_mcdi_filter_get_unsafe_id(u32 filter_id)41 static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id)
42 {
43 WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID);
44 return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1);
45 }
46
efx_mcdi_filter_get_unsafe_pri(u32 filter_id)47 static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id)
48 {
49 return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2);
50 }
51
efx_mcdi_filter_make_filter_id(unsigned int pri,u16 idx)52 static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx)
53 {
54 return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx;
55 }
56
57 /*
58 * Decide whether a filter should be exclusive or else should allow
59 * delivery to additional recipients. Currently we decide that
60 * filters for specific local unicast MAC and IP addresses are
61 * exclusive.
62 */
efx_mcdi_filter_is_exclusive(const struct efx_filter_spec * spec)63 static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec)
64 {
65 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
66 !is_multicast_ether_addr(spec->loc_mac))
67 return true;
68
69 if ((spec->match_flags &
70 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
71 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
72 if (spec->ether_type == htons(ETH_P_IP) &&
73 !ipv4_is_multicast(spec->loc_host[0]))
74 return true;
75 if (spec->ether_type == htons(ETH_P_IPV6) &&
76 ((const u8 *)spec->loc_host)[0] != 0xff)
77 return true;
78 }
79
80 return false;
81 }
82
83 static void
efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table * table,unsigned int filter_idx,const struct efx_filter_spec * spec,unsigned int flags)84 efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table,
85 unsigned int filter_idx,
86 const struct efx_filter_spec *spec,
87 unsigned int flags)
88 {
89 table->entry[filter_idx].spec = (unsigned long)spec | flags;
90 }
91
92 static void
efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic * efx,const struct efx_filter_spec * spec,efx_dword_t * inbuf)93 efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx,
94 const struct efx_filter_spec *spec,
95 efx_dword_t *inbuf)
96 {
97 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
98 u32 match_fields = 0, uc_match, mc_match;
99
100 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
101 efx_mcdi_filter_is_exclusive(spec) ?
102 MC_CMD_FILTER_OP_IN_OP_INSERT :
103 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);
104
105 /*
106 * Convert match flags and values. Unlike almost
107 * everything else in MCDI, these fields are in
108 * network byte order.
109 */
110 #define COPY_VALUE(value, mcdi_field) \
111 do { \
112 match_fields |= \
113 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \
114 mcdi_field ## _LBN; \
115 BUILD_BUG_ON( \
116 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
117 sizeof(value)); \
118 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
119 &value, sizeof(value)); \
120 } while (0)
121 #define COPY_FIELD(gen_flag, gen_field, mcdi_field) \
122 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \
123 COPY_VALUE(spec->gen_field, mcdi_field); \
124 }
125 /*
126 * Handle encap filters first. They will always be mismatch
127 * (unknown UC or MC) filters
128 */
129 if (encap_type) {
130 /*
131 * ether_type and outer_ip_proto need to be variables
132 * because COPY_VALUE wants to memcpy them
133 */
134 __be16 ether_type =
135 htons(encap_type & EFX_ENCAP_FLAG_IPV6 ?
136 ETH_P_IPV6 : ETH_P_IP);
137 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE;
138 u8 outer_ip_proto;
139
140 switch (encap_type & EFX_ENCAP_TYPES_MASK) {
141 case EFX_ENCAP_TYPE_VXLAN:
142 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN;
143 fallthrough;
144 case EFX_ENCAP_TYPE_GENEVE:
145 COPY_VALUE(ether_type, ETHER_TYPE);
146 outer_ip_proto = IPPROTO_UDP;
147 COPY_VALUE(outer_ip_proto, IP_PROTO);
148 /*
149 * We always need to set the type field, even
150 * though we're not matching on the TNI.
151 */
152 MCDI_POPULATE_DWORD_1(inbuf,
153 FILTER_OP_EXT_IN_VNI_OR_VSID,
154 FILTER_OP_EXT_IN_VNI_TYPE,
155 vni_type);
156 break;
157 case EFX_ENCAP_TYPE_NVGRE:
158 COPY_VALUE(ether_type, ETHER_TYPE);
159 outer_ip_proto = IPPROTO_GRE;
160 COPY_VALUE(outer_ip_proto, IP_PROTO);
161 break;
162 default:
163 WARN_ON(1);
164 }
165
166 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
167 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
168 } else {
169 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
170 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
171 }
172
173 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
174 match_fields |=
175 is_multicast_ether_addr(spec->loc_mac) ?
176 1 << mc_match :
177 1 << uc_match;
178 COPY_FIELD(REM_HOST, rem_host, SRC_IP);
179 COPY_FIELD(LOC_HOST, loc_host, DST_IP);
180 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
181 COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
182 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
183 COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
184 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
185 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
186 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
187 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
188 #undef COPY_FIELD
189 #undef COPY_VALUE
190 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
191 match_fields);
192 }
193
efx_mcdi_filter_push_prep(struct efx_nic * efx,const struct efx_filter_spec * spec,efx_dword_t * inbuf,u64 handle,struct efx_rss_context * ctx,bool replacing)194 static void efx_mcdi_filter_push_prep(struct efx_nic *efx,
195 const struct efx_filter_spec *spec,
196 efx_dword_t *inbuf, u64 handle,
197 struct efx_rss_context *ctx,
198 bool replacing)
199 {
200 u32 flags = spec->flags;
201
202 memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN);
203
204 /* If RSS filter, caller better have given us an RSS context */
205 if (flags & EFX_FILTER_FLAG_RX_RSS) {
206 /*
207 * We don't have the ability to return an error, so we'll just
208 * log a warning and disable RSS for the filter.
209 */
210 if (WARN_ON_ONCE(!ctx))
211 flags &= ~EFX_FILTER_FLAG_RX_RSS;
212 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID))
213 flags &= ~EFX_FILTER_FLAG_RX_RSS;
214 }
215
216 if (replacing) {
217 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
218 MC_CMD_FILTER_OP_IN_OP_REPLACE);
219 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
220 } else {
221 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf);
222 }
223
224 if (flags & EFX_FILTER_FLAG_VPORT_ID)
225 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, spec->vport_id);
226 else
227 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id);
228 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
229 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
230 MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
231 MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
232 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0);
233 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
234 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
235 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE,
236 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
237 0 : spec->dmaq_id);
238 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
239 (flags & EFX_FILTER_FLAG_RX_RSS) ?
240 MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
241 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
242 if (flags & EFX_FILTER_FLAG_RX_RSS)
243 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id);
244 }
245
efx_mcdi_filter_push(struct efx_nic * efx,const struct efx_filter_spec * spec,u64 * handle,struct efx_rss_context * ctx,bool replacing)246 static int efx_mcdi_filter_push(struct efx_nic *efx,
247 const struct efx_filter_spec *spec, u64 *handle,
248 struct efx_rss_context *ctx, bool replacing)
249 {
250 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
251 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN);
252 size_t outlen;
253 int rc;
254
255 efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing);
256 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
257 outbuf, sizeof(outbuf), &outlen);
258 if (rc && spec->priority != EFX_FILTER_PRI_HINT)
259 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf),
260 outbuf, outlen, rc);
261 if (rc == 0)
262 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
263 if (rc == -ENOSPC)
264 rc = -EBUSY; /* to match efx_farch_filter_insert() */
265 return rc;
266 }
267
efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec * spec)268 static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
269 {
270 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
271 unsigned int match_flags = spec->match_flags;
272 unsigned int uc_match, mc_match;
273 u32 mcdi_flags = 0;
274
275 #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \
276 unsigned int old_match_flags = match_flags; \
277 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \
278 if (match_flags != old_match_flags) \
279 mcdi_flags |= \
280 (1 << ((encap) ? \
281 MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \
282 mcdi_field ## _LBN : \
283 MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\
284 mcdi_field ## _LBN)); \
285 }
286 /* inner or outer based on encap type */
287 MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type);
288 MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type);
289 MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type);
290 MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type);
291 MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type);
292 MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type);
293 MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type);
294 MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type);
295 /* always outer */
296 MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false);
297 MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false);
298 #undef MAP_FILTER_TO_MCDI_FLAG
299
300 /* special handling for encap type, and mismatch */
301 if (encap_type) {
302 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE;
303 mcdi_flags |=
304 (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
305 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
306
307 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
308 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
309 } else {
310 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
311 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
312 }
313
314 if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
315 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
316 mcdi_flags |=
317 is_multicast_ether_addr(spec->loc_mac) ?
318 1 << mc_match :
319 1 << uc_match;
320 }
321
322 /* Did we map them all? */
323 WARN_ON_ONCE(match_flags);
324
325 return mcdi_flags;
326 }
327
efx_mcdi_filter_pri(struct efx_mcdi_filter_table * table,const struct efx_filter_spec * spec)328 static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table,
329 const struct efx_filter_spec *spec)
330 {
331 u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
332 unsigned int match_pri;
333
334 for (match_pri = 0;
335 match_pri < table->rx_match_count;
336 match_pri++)
337 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
338 return match_pri;
339
340 return -EPROTONOSUPPORT;
341 }
342
efx_mcdi_filter_insert_locked(struct efx_nic * efx,struct efx_filter_spec * spec,bool replace_equal)343 static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx,
344 struct efx_filter_spec *spec,
345 bool replace_equal)
346 {
347 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
348 struct efx_mcdi_filter_table *table;
349 struct efx_filter_spec *saved_spec;
350 struct efx_rss_context *ctx = NULL;
351 unsigned int match_pri, hash;
352 unsigned int priv_flags;
353 bool rss_locked = false;
354 bool replacing = false;
355 unsigned int depth, i;
356 int ins_index = -1;
357 DEFINE_WAIT(wait);
358 bool is_mc_recip;
359 s32 rc;
360
361 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
362 table = efx->filter_state;
363 down_write(&table->lock);
364
365 /* For now, only support RX filters */
366 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
367 EFX_FILTER_FLAG_RX) {
368 rc = -EINVAL;
369 goto out_unlock;
370 }
371
372 rc = efx_mcdi_filter_pri(table, spec);
373 if (rc < 0)
374 goto out_unlock;
375 match_pri = rc;
376
377 hash = efx_filter_spec_hash(spec);
378 is_mc_recip = efx_filter_is_mc_recipient(spec);
379 if (is_mc_recip)
380 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
381
382 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
383 mutex_lock(&efx->rss_lock);
384 rss_locked = true;
385 if (spec->rss_context)
386 ctx = efx_find_rss_context_entry(efx, spec->rss_context);
387 else
388 ctx = &efx->rss_context;
389 if (!ctx) {
390 rc = -ENOENT;
391 goto out_unlock;
392 }
393 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
394 rc = -EOPNOTSUPP;
395 goto out_unlock;
396 }
397 }
398
399 /* Find any existing filters with the same match tuple or
400 * else a free slot to insert at.
401 */
402 for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
403 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
404 saved_spec = efx_mcdi_filter_entry_spec(table, i);
405
406 if (!saved_spec) {
407 if (ins_index < 0)
408 ins_index = i;
409 } else if (efx_filter_spec_equal(spec, saved_spec)) {
410 if (spec->priority < saved_spec->priority &&
411 spec->priority != EFX_FILTER_PRI_AUTO) {
412 rc = -EPERM;
413 goto out_unlock;
414 }
415 if (!is_mc_recip) {
416 /* This is the only one */
417 if (spec->priority ==
418 saved_spec->priority &&
419 !replace_equal) {
420 rc = -EEXIST;
421 goto out_unlock;
422 }
423 ins_index = i;
424 break;
425 } else if (spec->priority >
426 saved_spec->priority ||
427 (spec->priority ==
428 saved_spec->priority &&
429 replace_equal)) {
430 if (ins_index < 0)
431 ins_index = i;
432 else
433 __set_bit(depth, mc_rem_map);
434 }
435 }
436 }
437
438 /* Once we reach the maximum search depth, use the first suitable
439 * slot, or return -EBUSY if there was none
440 */
441 if (ins_index < 0) {
442 rc = -EBUSY;
443 goto out_unlock;
444 }
445
446 /* Create a software table entry if necessary. */
447 saved_spec = efx_mcdi_filter_entry_spec(table, ins_index);
448 if (saved_spec) {
449 if (spec->priority == EFX_FILTER_PRI_AUTO &&
450 saved_spec->priority >= EFX_FILTER_PRI_AUTO) {
451 /* Just make sure it won't be removed */
452 if (saved_spec->priority > EFX_FILTER_PRI_AUTO)
453 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
454 table->entry[ins_index].spec &=
455 ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
456 rc = ins_index;
457 goto out_unlock;
458 }
459 replacing = true;
460 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index);
461 } else {
462 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
463 if (!saved_spec) {
464 rc = -ENOMEM;
465 goto out_unlock;
466 }
467 *saved_spec = *spec;
468 priv_flags = 0;
469 }
470 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
471
472 /* Actually insert the filter on the HW */
473 rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle,
474 ctx, replacing);
475
476 if (rc == -EINVAL && efx->must_realloc_vis)
477 /* The MC rebooted under us, causing it to reject our filter
478 * insertion as pointing to an invalid VI (spec->dmaq_id).
479 */
480 rc = -EAGAIN;
481
482 /* Finalise the software table entry */
483 if (rc == 0) {
484 if (replacing) {
485 /* Update the fields that may differ */
486 if (saved_spec->priority == EFX_FILTER_PRI_AUTO)
487 saved_spec->flags |=
488 EFX_FILTER_FLAG_RX_OVER_AUTO;
489 saved_spec->priority = spec->priority;
490 saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO;
491 saved_spec->flags |= spec->flags;
492 saved_spec->rss_context = spec->rss_context;
493 saved_spec->dmaq_id = spec->dmaq_id;
494 saved_spec->vport_id = spec->vport_id;
495 }
496 } else if (!replacing) {
497 kfree(saved_spec);
498 saved_spec = NULL;
499 } else {
500 /* We failed to replace, so the old filter is still present.
501 * Roll back the software table to reflect this. In fact the
502 * efx_mcdi_filter_set_entry() call below will do the right
503 * thing, so nothing extra is needed here.
504 */
505 }
506 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
507
508 /* Remove and finalise entries for lower-priority multicast
509 * recipients
510 */
511 if (is_mc_recip) {
512 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
513 unsigned int depth, i;
514
515 memset(inbuf, 0, sizeof(inbuf));
516
517 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
518 if (!test_bit(depth, mc_rem_map))
519 continue;
520
521 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
522 saved_spec = efx_mcdi_filter_entry_spec(table, i);
523 priv_flags = efx_mcdi_filter_entry_flags(table, i);
524
525 if (rc == 0) {
526 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
527 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
528 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
529 table->entry[i].handle);
530 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
531 inbuf, sizeof(inbuf),
532 NULL, 0, NULL);
533 }
534
535 if (rc == 0) {
536 kfree(saved_spec);
537 saved_spec = NULL;
538 priv_flags = 0;
539 }
540 efx_mcdi_filter_set_entry(table, i, saved_spec,
541 priv_flags);
542 }
543 }
544
545 /* If successful, return the inserted filter ID */
546 if (rc == 0)
547 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index);
548
549 out_unlock:
550 if (rss_locked)
551 mutex_unlock(&efx->rss_lock);
552 up_write(&table->lock);
553 return rc;
554 }
555
efx_mcdi_filter_insert(struct efx_nic * efx,struct efx_filter_spec * spec,bool replace_equal)556 s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec,
557 bool replace_equal)
558 {
559 s32 ret;
560
561 down_read(&efx->filter_sem);
562 ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal);
563 up_read(&efx->filter_sem);
564
565 return ret;
566 }
567
568 /*
569 * Remove a filter.
570 * If !by_index, remove by ID
571 * If by_index, remove by index
572 * Filter ID may come from userland and must be range-checked.
573 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
574 * for write.
575 */
efx_mcdi_filter_remove_internal(struct efx_nic * efx,unsigned int priority_mask,u32 filter_id,bool by_index)576 static int efx_mcdi_filter_remove_internal(struct efx_nic *efx,
577 unsigned int priority_mask,
578 u32 filter_id, bool by_index)
579 {
580 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
581 struct efx_mcdi_filter_table *table = efx->filter_state;
582 MCDI_DECLARE_BUF(inbuf,
583 MC_CMD_FILTER_OP_IN_HANDLE_OFST +
584 MC_CMD_FILTER_OP_IN_HANDLE_LEN);
585 struct efx_filter_spec *spec;
586 DEFINE_WAIT(wait);
587 int rc;
588
589 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
590 if (!spec ||
591 (!by_index &&
592 efx_mcdi_filter_pri(table, spec) !=
593 efx_mcdi_filter_get_unsafe_pri(filter_id)))
594 return -ENOENT;
595
596 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
597 priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
598 /* Just remove flags */
599 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
600 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
601 return 0;
602 }
603
604 if (!(priority_mask & (1U << spec->priority)))
605 return -ENOENT;
606
607 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
608 /* Reset to an automatic filter */
609
610 struct efx_filter_spec new_spec = *spec;
611
612 new_spec.priority = EFX_FILTER_PRI_AUTO;
613 new_spec.flags = (EFX_FILTER_FLAG_RX |
614 (efx_rss_active(&efx->rss_context) ?
615 EFX_FILTER_FLAG_RX_RSS : 0));
616 new_spec.dmaq_id = 0;
617 new_spec.rss_context = 0;
618 rc = efx_mcdi_filter_push(efx, &new_spec,
619 &table->entry[filter_idx].handle,
620 &efx->rss_context,
621 true);
622
623 if (rc == 0)
624 *spec = new_spec;
625 } else {
626 /* Really remove the filter */
627
628 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
629 efx_mcdi_filter_is_exclusive(spec) ?
630 MC_CMD_FILTER_OP_IN_OP_REMOVE :
631 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
632 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
633 table->entry[filter_idx].handle);
634 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
635 inbuf, sizeof(inbuf), NULL, 0, NULL);
636
637 if ((rc == 0) || (rc == -ENOENT)) {
638 /* Filter removed OK or didn't actually exist */
639 kfree(spec);
640 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
641 } else {
642 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP,
643 MC_CMD_FILTER_OP_EXT_IN_LEN,
644 NULL, 0, rc);
645 }
646 }
647
648 return rc;
649 }
650
651 /* Remove filters that weren't renewed. */
efx_mcdi_filter_remove_old(struct efx_nic * efx)652 static void efx_mcdi_filter_remove_old(struct efx_nic *efx)
653 {
654 struct efx_mcdi_filter_table *table = efx->filter_state;
655 int remove_failed = 0;
656 int remove_noent = 0;
657 int rc;
658 int i;
659
660 down_write(&table->lock);
661 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
662 if (READ_ONCE(table->entry[i].spec) &
663 EFX_EF10_FILTER_FLAG_AUTO_OLD) {
664 rc = efx_mcdi_filter_remove_internal(efx,
665 1U << EFX_FILTER_PRI_AUTO, i, true);
666 if (rc == -ENOENT)
667 remove_noent++;
668 else if (rc)
669 remove_failed++;
670 }
671 }
672 up_write(&table->lock);
673
674 if (remove_failed)
675 netif_info(efx, drv, efx->net_dev,
676 "%s: failed to remove %d filters\n",
677 __func__, remove_failed);
678 if (remove_noent)
679 netif_info(efx, drv, efx->net_dev,
680 "%s: failed to remove %d non-existent filters\n",
681 __func__, remove_noent);
682 }
683
efx_mcdi_filter_remove_safe(struct efx_nic * efx,enum efx_filter_priority priority,u32 filter_id)684 int efx_mcdi_filter_remove_safe(struct efx_nic *efx,
685 enum efx_filter_priority priority,
686 u32 filter_id)
687 {
688 struct efx_mcdi_filter_table *table;
689 int rc;
690
691 down_read(&efx->filter_sem);
692 table = efx->filter_state;
693 down_write(&table->lock);
694 rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
695 false);
696 up_write(&table->lock);
697 up_read(&efx->filter_sem);
698 return rc;
699 }
700
701 /* Caller must hold efx->filter_sem for read */
efx_mcdi_filter_remove_unsafe(struct efx_nic * efx,enum efx_filter_priority priority,u32 filter_id)702 static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx,
703 enum efx_filter_priority priority,
704 u32 filter_id)
705 {
706 struct efx_mcdi_filter_table *table = efx->filter_state;
707
708 if (filter_id == EFX_EF10_FILTER_ID_INVALID)
709 return;
710
711 down_write(&table->lock);
712 efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
713 true);
714 up_write(&table->lock);
715 }
716
efx_mcdi_filter_get_safe(struct efx_nic * efx,enum efx_filter_priority priority,u32 filter_id,struct efx_filter_spec * spec)717 int efx_mcdi_filter_get_safe(struct efx_nic *efx,
718 enum efx_filter_priority priority,
719 u32 filter_id, struct efx_filter_spec *spec)
720 {
721 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
722 const struct efx_filter_spec *saved_spec;
723 struct efx_mcdi_filter_table *table;
724 int rc;
725
726 down_read(&efx->filter_sem);
727 table = efx->filter_state;
728 down_read(&table->lock);
729 saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx);
730 if (saved_spec && saved_spec->priority == priority &&
731 efx_mcdi_filter_pri(table, saved_spec) ==
732 efx_mcdi_filter_get_unsafe_pri(filter_id)) {
733 *spec = *saved_spec;
734 rc = 0;
735 } else {
736 rc = -ENOENT;
737 }
738 up_read(&table->lock);
739 up_read(&efx->filter_sem);
740 return rc;
741 }
742
efx_mcdi_filter_insert_addr_list(struct efx_nic * efx,struct efx_mcdi_filter_vlan * vlan,bool multicast,bool rollback)743 static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx,
744 struct efx_mcdi_filter_vlan *vlan,
745 bool multicast, bool rollback)
746 {
747 struct efx_mcdi_filter_table *table = efx->filter_state;
748 struct efx_mcdi_dev_addr *addr_list;
749 enum efx_filter_flags filter_flags;
750 struct efx_filter_spec spec;
751 u8 baddr[ETH_ALEN];
752 unsigned int i, j;
753 int addr_count;
754 u16 *ids;
755 int rc;
756
757 if (multicast) {
758 addr_list = table->dev_mc_list;
759 addr_count = table->dev_mc_count;
760 ids = vlan->mc;
761 } else {
762 addr_list = table->dev_uc_list;
763 addr_count = table->dev_uc_count;
764 ids = vlan->uc;
765 }
766
767 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
768
769 /* Insert/renew filters */
770 for (i = 0; i < addr_count; i++) {
771 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID);
772 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
773 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
774 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
775 if (rc < 0) {
776 if (rollback) {
777 netif_info(efx, drv, efx->net_dev,
778 "efx_mcdi_filter_insert failed rc=%d\n",
779 rc);
780 /* Fall back to promiscuous */
781 for (j = 0; j < i; j++) {
782 efx_mcdi_filter_remove_unsafe(
783 efx, EFX_FILTER_PRI_AUTO,
784 ids[j]);
785 ids[j] = EFX_EF10_FILTER_ID_INVALID;
786 }
787 return rc;
788 } else {
789 /* keep invalid ID, and carry on */
790 }
791 } else {
792 ids[i] = efx_mcdi_filter_get_unsafe_id(rc);
793 }
794 }
795
796 if (multicast && rollback) {
797 /* Also need an Ethernet broadcast filter */
798 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] !=
799 EFX_EF10_FILTER_ID_INVALID);
800 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
801 eth_broadcast_addr(baddr);
802 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
803 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
804 if (rc < 0) {
805 netif_warn(efx, drv, efx->net_dev,
806 "Broadcast filter insert failed rc=%d\n", rc);
807 /* Fall back to promiscuous */
808 for (j = 0; j < i; j++) {
809 efx_mcdi_filter_remove_unsafe(
810 efx, EFX_FILTER_PRI_AUTO,
811 ids[j]);
812 ids[j] = EFX_EF10_FILTER_ID_INVALID;
813 }
814 return rc;
815 } else {
816 vlan->default_filters[EFX_EF10_BCAST] =
817 efx_mcdi_filter_get_unsafe_id(rc);
818 }
819 }
820
821 return 0;
822 }
823
efx_mcdi_filter_insert_def(struct efx_nic * efx,struct efx_mcdi_filter_vlan * vlan,enum efx_encap_type encap_type,bool multicast,bool rollback)824 static int efx_mcdi_filter_insert_def(struct efx_nic *efx,
825 struct efx_mcdi_filter_vlan *vlan,
826 enum efx_encap_type encap_type,
827 bool multicast, bool rollback)
828 {
829 struct efx_mcdi_filter_table *table = efx->filter_state;
830 enum efx_filter_flags filter_flags;
831 struct efx_filter_spec spec;
832 u8 baddr[ETH_ALEN];
833 int rc;
834 u16 *id;
835
836 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
837
838 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
839
840 if (multicast)
841 efx_filter_set_mc_def(&spec);
842 else
843 efx_filter_set_uc_def(&spec);
844
845 if (encap_type) {
846 if (efx_has_cap(efx, VXLAN_NVGRE))
847 efx_filter_set_encap_type(&spec, encap_type);
848 else
849 /*
850 * don't insert encap filters on non-supporting
851 * platforms. ID will be left as INVALID.
852 */
853 return 0;
854 }
855
856 if (vlan->vid != EFX_FILTER_VID_UNSPEC)
857 efx_filter_set_eth_local(&spec, vlan->vid, NULL);
858
859 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
860 if (rc < 0) {
861 const char *um = multicast ? "Multicast" : "Unicast";
862 const char *encap_name = "";
863 const char *encap_ipv = "";
864
865 if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
866 EFX_ENCAP_TYPE_VXLAN)
867 encap_name = "VXLAN ";
868 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
869 EFX_ENCAP_TYPE_NVGRE)
870 encap_name = "NVGRE ";
871 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
872 EFX_ENCAP_TYPE_GENEVE)
873 encap_name = "GENEVE ";
874 if (encap_type & EFX_ENCAP_FLAG_IPV6)
875 encap_ipv = "IPv6 ";
876 else if (encap_type)
877 encap_ipv = "IPv4 ";
878
879 /*
880 * unprivileged functions can't insert mismatch filters
881 * for encapsulated or unicast traffic, so downgrade
882 * those warnings to debug.
883 */
884 netif_cond_dbg(efx, drv, efx->net_dev,
885 rc == -EPERM && (encap_type || !multicast), warn,
886 "%s%s%s mismatch filter insert failed rc=%d\n",
887 encap_name, encap_ipv, um, rc);
888 } else if (multicast) {
889 /* mapping from encap types to default filter IDs (multicast) */
890 static enum efx_mcdi_filter_default_filters map[] = {
891 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF,
892 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF,
893 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF,
894 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF,
895 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
896 EFX_EF10_VXLAN6_MCDEF,
897 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
898 EFX_EF10_NVGRE6_MCDEF,
899 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
900 EFX_EF10_GENEVE6_MCDEF,
901 };
902
903 /* quick bounds check (BCAST result impossible) */
904 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
905 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
906 WARN_ON(1);
907 return -EINVAL;
908 }
909 /* then follow map */
910 id = &vlan->default_filters[map[encap_type]];
911
912 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
913 *id = efx_mcdi_filter_get_unsafe_id(rc);
914 if (!table->mc_chaining && !encap_type) {
915 /* Also need an Ethernet broadcast filter */
916 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
917 filter_flags, 0);
918 eth_broadcast_addr(baddr);
919 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
920 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
921 if (rc < 0) {
922 netif_warn(efx, drv, efx->net_dev,
923 "Broadcast filter insert failed rc=%d\n",
924 rc);
925 if (rollback) {
926 /* Roll back the mc_def filter */
927 efx_mcdi_filter_remove_unsafe(
928 efx, EFX_FILTER_PRI_AUTO,
929 *id);
930 *id = EFX_EF10_FILTER_ID_INVALID;
931 return rc;
932 }
933 } else {
934 EFX_WARN_ON_PARANOID(
935 vlan->default_filters[EFX_EF10_BCAST] !=
936 EFX_EF10_FILTER_ID_INVALID);
937 vlan->default_filters[EFX_EF10_BCAST] =
938 efx_mcdi_filter_get_unsafe_id(rc);
939 }
940 }
941 rc = 0;
942 } else {
943 /* mapping from encap types to default filter IDs (unicast) */
944 static enum efx_mcdi_filter_default_filters map[] = {
945 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF,
946 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF,
947 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF,
948 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF,
949 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
950 EFX_EF10_VXLAN6_UCDEF,
951 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
952 EFX_EF10_NVGRE6_UCDEF,
953 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
954 EFX_EF10_GENEVE6_UCDEF,
955 };
956
957 /* quick bounds check (BCAST result impossible) */
958 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
959 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
960 WARN_ON(1);
961 return -EINVAL;
962 }
963 /* then follow map */
964 id = &vlan->default_filters[map[encap_type]];
965 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
966 *id = rc;
967 rc = 0;
968 }
969 return rc;
970 }
971
972 /*
973 * Caller must hold efx->filter_sem for read if race against
974 * efx_mcdi_filter_table_remove() is possible
975 */
efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic * efx,struct efx_mcdi_filter_vlan * vlan)976 static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx,
977 struct efx_mcdi_filter_vlan *vlan)
978 {
979 struct efx_mcdi_filter_table *table = efx->filter_state;
980
981 /*
982 * Do not install unspecified VID if VLAN filtering is enabled.
983 * Do not install all specified VIDs if VLAN filtering is disabled.
984 */
985 if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
986 return;
987
988 /* Insert/renew unicast filters */
989 if (table->uc_promisc) {
990 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE,
991 false, false);
992 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false);
993 } else {
994 /*
995 * If any of the filters failed to insert, fall back to
996 * promiscuous mode - add in the uc_def filter. But keep
997 * our individual unicast filters.
998 */
999 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false))
1000 efx_mcdi_filter_insert_def(efx, vlan,
1001 EFX_ENCAP_TYPE_NONE,
1002 false, false);
1003 }
1004 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1005 false, false);
1006 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1007 EFX_ENCAP_FLAG_IPV6,
1008 false, false);
1009 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1010 false, false);
1011 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1012 EFX_ENCAP_FLAG_IPV6,
1013 false, false);
1014 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1015 false, false);
1016 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1017 EFX_ENCAP_FLAG_IPV6,
1018 false, false);
1019
1020 /*
1021 * Insert/renew multicast filters
1022 *
1023 * If changing promiscuous state with cascaded multicast filters, remove
1024 * old filters first, so that packets are dropped rather than duplicated
1025 */
1026 if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc)
1027 efx_mcdi_filter_remove_old(efx);
1028 if (table->mc_promisc) {
1029 if (table->mc_chaining) {
1030 /*
1031 * If we failed to insert promiscuous filters, rollback
1032 * and fall back to individual multicast filters
1033 */
1034 if (efx_mcdi_filter_insert_def(efx, vlan,
1035 EFX_ENCAP_TYPE_NONE,
1036 true, true)) {
1037 /* Changing promisc state, so remove old filters */
1038 efx_mcdi_filter_remove_old(efx);
1039 efx_mcdi_filter_insert_addr_list(efx, vlan,
1040 true, false);
1041 }
1042 } else {
1043 /*
1044 * If we failed to insert promiscuous filters, don't
1045 * rollback. Regardless, also insert the mc_list,
1046 * unless it's incomplete due to overflow
1047 */
1048 efx_mcdi_filter_insert_def(efx, vlan,
1049 EFX_ENCAP_TYPE_NONE,
1050 true, false);
1051 if (!table->mc_overflow)
1052 efx_mcdi_filter_insert_addr_list(efx, vlan,
1053 true, false);
1054 }
1055 } else {
1056 /*
1057 * If any filters failed to insert, rollback and fall back to
1058 * promiscuous mode - mc_def filter and maybe broadcast. If
1059 * that fails, roll back again and insert as many of our
1060 * individual multicast filters as we can.
1061 */
1062 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) {
1063 /* Changing promisc state, so remove old filters */
1064 if (table->mc_chaining)
1065 efx_mcdi_filter_remove_old(efx);
1066 if (efx_mcdi_filter_insert_def(efx, vlan,
1067 EFX_ENCAP_TYPE_NONE,
1068 true, true))
1069 efx_mcdi_filter_insert_addr_list(efx, vlan,
1070 true, false);
1071 }
1072 }
1073 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1074 true, false);
1075 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1076 EFX_ENCAP_FLAG_IPV6,
1077 true, false);
1078 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1079 true, false);
1080 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1081 EFX_ENCAP_FLAG_IPV6,
1082 true, false);
1083 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1084 true, false);
1085 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1086 EFX_ENCAP_FLAG_IPV6,
1087 true, false);
1088 }
1089
efx_mcdi_filter_clear_rx(struct efx_nic * efx,enum efx_filter_priority priority)1090 int efx_mcdi_filter_clear_rx(struct efx_nic *efx,
1091 enum efx_filter_priority priority)
1092 {
1093 struct efx_mcdi_filter_table *table;
1094 unsigned int priority_mask;
1095 unsigned int i;
1096 int rc;
1097
1098 priority_mask = (((1U << (priority + 1)) - 1) &
1099 ~(1U << EFX_FILTER_PRI_AUTO));
1100
1101 down_read(&efx->filter_sem);
1102 table = efx->filter_state;
1103 down_write(&table->lock);
1104 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
1105 rc = efx_mcdi_filter_remove_internal(efx, priority_mask,
1106 i, true);
1107 if (rc && rc != -ENOENT)
1108 break;
1109 rc = 0;
1110 }
1111
1112 up_write(&table->lock);
1113 up_read(&efx->filter_sem);
1114 return rc;
1115 }
1116
efx_mcdi_filter_count_rx_used(struct efx_nic * efx,enum efx_filter_priority priority)1117 u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx,
1118 enum efx_filter_priority priority)
1119 {
1120 struct efx_mcdi_filter_table *table;
1121 unsigned int filter_idx;
1122 s32 count = 0;
1123
1124 down_read(&efx->filter_sem);
1125 table = efx->filter_state;
1126 down_read(&table->lock);
1127 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1128 if (table->entry[filter_idx].spec &&
1129 efx_mcdi_filter_entry_spec(table, filter_idx)->priority ==
1130 priority)
1131 ++count;
1132 }
1133 up_read(&table->lock);
1134 up_read(&efx->filter_sem);
1135 return count;
1136 }
1137
efx_mcdi_filter_get_rx_id_limit(struct efx_nic * efx)1138 u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx)
1139 {
1140 struct efx_mcdi_filter_table *table = efx->filter_state;
1141
1142 return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2;
1143 }
1144
efx_mcdi_filter_get_rx_ids(struct efx_nic * efx,enum efx_filter_priority priority,u32 * buf,u32 size)1145 s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx,
1146 enum efx_filter_priority priority,
1147 u32 *buf, u32 size)
1148 {
1149 struct efx_mcdi_filter_table *table;
1150 struct efx_filter_spec *spec;
1151 unsigned int filter_idx;
1152 s32 count = 0;
1153
1154 down_read(&efx->filter_sem);
1155 table = efx->filter_state;
1156 down_read(&table->lock);
1157
1158 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1159 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1160 if (spec && spec->priority == priority) {
1161 if (count == size) {
1162 count = -EMSGSIZE;
1163 break;
1164 }
1165 buf[count++] =
1166 efx_mcdi_filter_make_filter_id(
1167 efx_mcdi_filter_pri(table, spec),
1168 filter_idx);
1169 }
1170 }
1171 up_read(&table->lock);
1172 up_read(&efx->filter_sem);
1173 return count;
1174 }
1175
efx_mcdi_filter_match_flags_from_mcdi(bool encap,u32 mcdi_flags)1176 static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
1177 {
1178 int match_flags = 0;
1179
1180 #define MAP_FLAG(gen_flag, mcdi_field) do { \
1181 u32 old_mcdi_flags = mcdi_flags; \
1182 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \
1183 mcdi_field ## _LBN); \
1184 if (mcdi_flags != old_mcdi_flags) \
1185 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \
1186 } while (0)
1187
1188 if (encap) {
1189 /* encap filters must specify encap type */
1190 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1191 /* and imply ethertype and ip proto */
1192 mcdi_flags &=
1193 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
1194 mcdi_flags &=
1195 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
1196 /* VLAN tags refer to the outer packet */
1197 MAP_FLAG(INNER_VID, INNER_VLAN);
1198 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1199 /* everything else refers to the inner packet */
1200 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST);
1201 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST);
1202 MAP_FLAG(REM_HOST, IFRM_SRC_IP);
1203 MAP_FLAG(LOC_HOST, IFRM_DST_IP);
1204 MAP_FLAG(REM_MAC, IFRM_SRC_MAC);
1205 MAP_FLAG(REM_PORT, IFRM_SRC_PORT);
1206 MAP_FLAG(LOC_MAC, IFRM_DST_MAC);
1207 MAP_FLAG(LOC_PORT, IFRM_DST_PORT);
1208 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE);
1209 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO);
1210 } else {
1211 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
1212 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
1213 MAP_FLAG(REM_HOST, SRC_IP);
1214 MAP_FLAG(LOC_HOST, DST_IP);
1215 MAP_FLAG(REM_MAC, SRC_MAC);
1216 MAP_FLAG(REM_PORT, SRC_PORT);
1217 MAP_FLAG(LOC_MAC, DST_MAC);
1218 MAP_FLAG(LOC_PORT, DST_PORT);
1219 MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
1220 MAP_FLAG(INNER_VID, INNER_VLAN);
1221 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1222 MAP_FLAG(IP_PROTO, IP_PROTO);
1223 }
1224 #undef MAP_FLAG
1225
1226 /* Did we map them all? */
1227 if (mcdi_flags)
1228 return -EINVAL;
1229
1230 return match_flags;
1231 }
1232
efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table * table,bool encap,enum efx_filter_match_flags match_flags)1233 bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table,
1234 bool encap,
1235 enum efx_filter_match_flags match_flags)
1236 {
1237 unsigned int match_pri;
1238 int mf;
1239
1240 for (match_pri = 0;
1241 match_pri < table->rx_match_count;
1242 match_pri++) {
1243 mf = efx_mcdi_filter_match_flags_from_mcdi(encap,
1244 table->rx_match_mcdi_flags[match_pri]);
1245 if (mf == match_flags)
1246 return true;
1247 }
1248
1249 return false;
1250 }
1251
1252 static int
efx_mcdi_filter_table_probe_matches(struct efx_nic * efx,struct efx_mcdi_filter_table * table,bool encap)1253 efx_mcdi_filter_table_probe_matches(struct efx_nic *efx,
1254 struct efx_mcdi_filter_table *table,
1255 bool encap)
1256 {
1257 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
1258 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1259 unsigned int pd_match_pri, pd_match_count;
1260 size_t outlen;
1261 int rc;
1262
1263 /* Find out which RX filter types are supported, and their priorities */
1264 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
1265 encap ?
1266 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1267 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1268 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
1269 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
1270 &outlen);
1271 if (rc)
1272 return rc;
1273
1274 pd_match_count = MCDI_VAR_ARRAY_LEN(
1275 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);
1276
1277 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
1278 u32 mcdi_flags =
1279 MCDI_ARRAY_DWORD(
1280 outbuf,
1281 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
1282 pd_match_pri);
1283 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags);
1284 if (rc < 0) {
1285 netif_dbg(efx, probe, efx->net_dev,
1286 "%s: fw flags %#x pri %u not supported in driver\n",
1287 __func__, mcdi_flags, pd_match_pri);
1288 } else {
1289 netif_dbg(efx, probe, efx->net_dev,
1290 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
1291 __func__, mcdi_flags, pd_match_pri,
1292 rc, table->rx_match_count);
1293 table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
1294 table->rx_match_count++;
1295 }
1296 }
1297
1298 return 0;
1299 }
1300
efx_mcdi_filter_table_probe(struct efx_nic * efx,bool multicast_chaining)1301 int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining)
1302 {
1303 struct net_device *net_dev = efx->net_dev;
1304 struct efx_mcdi_filter_table *table;
1305 int rc;
1306
1307 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1308 return -EINVAL;
1309
1310 if (efx->filter_state) /* already probed */
1311 return 0;
1312
1313 table = kzalloc(sizeof(*table), GFP_KERNEL);
1314 if (!table)
1315 return -ENOMEM;
1316
1317 table->mc_chaining = multicast_chaining;
1318 table->rx_match_count = 0;
1319 rc = efx_mcdi_filter_table_probe_matches(efx, table, false);
1320 if (rc)
1321 goto fail;
1322 if (efx_has_cap(efx, VXLAN_NVGRE))
1323 rc = efx_mcdi_filter_table_probe_matches(efx, table, true);
1324 if (rc)
1325 goto fail;
1326 if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1327 !(efx_mcdi_filter_match_supported(table, false,
1328 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
1329 efx_mcdi_filter_match_supported(table, false,
1330 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
1331 netif_info(efx, probe, net_dev,
1332 "VLAN filters are not supported in this firmware variant\n");
1333 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1334 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1335 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1336 }
1337
1338 table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS,
1339 sizeof(*table->entry)));
1340 if (!table->entry) {
1341 rc = -ENOMEM;
1342 goto fail;
1343 }
1344
1345 table->mc_promisc_last = false;
1346 table->vlan_filter =
1347 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1348 INIT_LIST_HEAD(&table->vlan_list);
1349 init_rwsem(&table->lock);
1350
1351 efx->filter_state = table;
1352
1353 return 0;
1354 fail:
1355 kfree(table);
1356 return rc;
1357 }
1358
efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic * efx)1359 void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx)
1360 {
1361 struct efx_mcdi_filter_table *table = efx->filter_state;
1362
1363 if (table) {
1364 table->must_restore_filters = true;
1365 table->must_restore_rss_contexts = true;
1366 }
1367 }
1368
1369 /*
1370 * Caller must hold efx->filter_sem for read if race against
1371 * efx_mcdi_filter_table_remove() is possible
1372 */
efx_mcdi_filter_table_restore(struct efx_nic * efx)1373 void efx_mcdi_filter_table_restore(struct efx_nic *efx)
1374 {
1375 struct efx_mcdi_filter_table *table = efx->filter_state;
1376 unsigned int invalid_filters = 0, failed = 0;
1377 struct efx_mcdi_filter_vlan *vlan;
1378 struct efx_filter_spec *spec;
1379 struct efx_rss_context *ctx;
1380 unsigned int filter_idx;
1381 u32 mcdi_flags;
1382 int match_pri;
1383 int rc, i;
1384
1385 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1386
1387 if (!table || !table->must_restore_filters)
1388 return;
1389
1390 down_write(&table->lock);
1391 mutex_lock(&efx->rss_lock);
1392
1393 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1394 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1395 if (!spec)
1396 continue;
1397
1398 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
1399 match_pri = 0;
1400 while (match_pri < table->rx_match_count &&
1401 table->rx_match_mcdi_flags[match_pri] != mcdi_flags)
1402 ++match_pri;
1403 if (match_pri >= table->rx_match_count) {
1404 invalid_filters++;
1405 goto not_restored;
1406 }
1407 if (spec->rss_context)
1408 ctx = efx_find_rss_context_entry(efx, spec->rss_context);
1409 else
1410 ctx = &efx->rss_context;
1411 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
1412 if (!ctx) {
1413 netif_warn(efx, drv, efx->net_dev,
1414 "Warning: unable to restore a filter with nonexistent RSS context %u.\n",
1415 spec->rss_context);
1416 invalid_filters++;
1417 goto not_restored;
1418 }
1419 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
1420 netif_warn(efx, drv, efx->net_dev,
1421 "Warning: unable to restore a filter with RSS context %u as it was not created.\n",
1422 spec->rss_context);
1423 invalid_filters++;
1424 goto not_restored;
1425 }
1426 }
1427
1428 rc = efx_mcdi_filter_push(efx, spec,
1429 &table->entry[filter_idx].handle,
1430 ctx, false);
1431 if (rc)
1432 failed++;
1433
1434 if (rc) {
1435 not_restored:
1436 list_for_each_entry(vlan, &table->vlan_list, list)
1437 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i)
1438 if (vlan->default_filters[i] == filter_idx)
1439 vlan->default_filters[i] =
1440 EFX_EF10_FILTER_ID_INVALID;
1441
1442 kfree(spec);
1443 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
1444 }
1445 }
1446
1447 mutex_unlock(&efx->rss_lock);
1448 up_write(&table->lock);
1449
1450 /*
1451 * This can happen validly if the MC's capabilities have changed, so
1452 * is not an error.
1453 */
1454 if (invalid_filters)
1455 netif_dbg(efx, drv, efx->net_dev,
1456 "Did not restore %u filters that are now unsupported.\n",
1457 invalid_filters);
1458
1459 if (failed)
1460 netif_err(efx, hw, efx->net_dev,
1461 "unable to restore %u filters\n", failed);
1462 else
1463 table->must_restore_filters = false;
1464 }
1465
efx_mcdi_filter_table_down(struct efx_nic * efx)1466 void efx_mcdi_filter_table_down(struct efx_nic *efx)
1467 {
1468 struct efx_mcdi_filter_table *table = efx->filter_state;
1469 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
1470 struct efx_filter_spec *spec;
1471 unsigned int filter_idx;
1472 int rc;
1473
1474 if (!table)
1475 return;
1476
1477 efx_mcdi_filter_cleanup_vlans(efx);
1478
1479 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1480 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1481 if (!spec)
1482 continue;
1483
1484 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1485 efx_mcdi_filter_is_exclusive(spec) ?
1486 MC_CMD_FILTER_OP_IN_OP_REMOVE :
1487 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
1488 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
1489 table->entry[filter_idx].handle);
1490 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf,
1491 sizeof(inbuf), NULL, 0, NULL);
1492 if (rc)
1493 netif_info(efx, drv, efx->net_dev,
1494 "%s: filter %04x remove failed\n",
1495 __func__, filter_idx);
1496 kfree(spec);
1497 }
1498 }
1499
efx_mcdi_filter_table_remove(struct efx_nic * efx)1500 void efx_mcdi_filter_table_remove(struct efx_nic *efx)
1501 {
1502 struct efx_mcdi_filter_table *table = efx->filter_state;
1503
1504 efx_mcdi_filter_table_down(efx);
1505
1506 efx->filter_state = NULL;
1507 /*
1508 * If we were called without locking, then it's not safe to free
1509 * the table as others might be using it. So we just WARN, leak
1510 * the memory, and potentially get an inconsistent filter table
1511 * state.
1512 * This should never actually happen.
1513 */
1514 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1515 return;
1516
1517 if (!table)
1518 return;
1519
1520 vfree(table->entry);
1521 kfree(table);
1522 }
1523
efx_mcdi_filter_mark_one_old(struct efx_nic * efx,uint16_t * id)1524 static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
1525 {
1526 struct efx_mcdi_filter_table *table = efx->filter_state;
1527 unsigned int filter_idx;
1528
1529 efx_rwsem_assert_write_locked(&table->lock);
1530
1531 if (*id != EFX_EF10_FILTER_ID_INVALID) {
1532 filter_idx = efx_mcdi_filter_get_unsafe_id(*id);
1533 if (!table->entry[filter_idx].spec)
1534 netif_dbg(efx, drv, efx->net_dev,
1535 "marked null spec old %04x:%04x\n", *id,
1536 filter_idx);
1537 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
1538 *id = EFX_EF10_FILTER_ID_INVALID;
1539 }
1540 }
1541
1542 /* Mark old per-VLAN filters that may need to be removed */
_efx_mcdi_filter_vlan_mark_old(struct efx_nic * efx,struct efx_mcdi_filter_vlan * vlan)1543 static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx,
1544 struct efx_mcdi_filter_vlan *vlan)
1545 {
1546 struct efx_mcdi_filter_table *table = efx->filter_state;
1547 unsigned int i;
1548
1549 for (i = 0; i < table->dev_uc_count; i++)
1550 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]);
1551 for (i = 0; i < table->dev_mc_count; i++)
1552 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]);
1553 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1554 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]);
1555 }
1556
1557 /*
1558 * Mark old filters that may need to be removed.
1559 * Caller must hold efx->filter_sem for read if race against
1560 * efx_mcdi_filter_table_remove() is possible
1561 */
efx_mcdi_filter_mark_old(struct efx_nic * efx)1562 static void efx_mcdi_filter_mark_old(struct efx_nic *efx)
1563 {
1564 struct efx_mcdi_filter_table *table = efx->filter_state;
1565 struct efx_mcdi_filter_vlan *vlan;
1566
1567 down_write(&table->lock);
1568 list_for_each_entry(vlan, &table->vlan_list, list)
1569 _efx_mcdi_filter_vlan_mark_old(efx, vlan);
1570 up_write(&table->lock);
1571 }
1572
efx_mcdi_filter_add_vlan(struct efx_nic * efx,u16 vid)1573 int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid)
1574 {
1575 struct efx_mcdi_filter_table *table = efx->filter_state;
1576 struct efx_mcdi_filter_vlan *vlan;
1577 unsigned int i;
1578
1579 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1580 return -EINVAL;
1581
1582 vlan = efx_mcdi_filter_find_vlan(efx, vid);
1583 if (WARN_ON(vlan)) {
1584 netif_err(efx, drv, efx->net_dev,
1585 "VLAN %u already added\n", vid);
1586 return -EALREADY;
1587 }
1588
1589 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
1590 if (!vlan)
1591 return -ENOMEM;
1592
1593 vlan->vid = vid;
1594
1595 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1596 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
1597 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1598 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
1599 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1600 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID;
1601
1602 list_add_tail(&vlan->list, &table->vlan_list);
1603
1604 if (efx_dev_registered(efx))
1605 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1606
1607 return 0;
1608 }
1609
efx_mcdi_filter_del_vlan_internal(struct efx_nic * efx,struct efx_mcdi_filter_vlan * vlan)1610 static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx,
1611 struct efx_mcdi_filter_vlan *vlan)
1612 {
1613 unsigned int i;
1614
1615 /* See comment in efx_mcdi_filter_table_remove() */
1616 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1617 return;
1618
1619 list_del(&vlan->list);
1620
1621 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1622 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1623 vlan->uc[i]);
1624 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1625 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1626 vlan->mc[i]);
1627 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1628 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID)
1629 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1630 vlan->default_filters[i]);
1631
1632 kfree(vlan);
1633 }
1634
efx_mcdi_filter_del_vlan(struct efx_nic * efx,u16 vid)1635 void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid)
1636 {
1637 struct efx_mcdi_filter_vlan *vlan;
1638
1639 /* See comment in efx_mcdi_filter_table_remove() */
1640 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1641 return;
1642
1643 vlan = efx_mcdi_filter_find_vlan(efx, vid);
1644 if (!vlan) {
1645 netif_err(efx, drv, efx->net_dev,
1646 "VLAN %u not found in filter state\n", vid);
1647 return;
1648 }
1649
1650 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1651 }
1652
efx_mcdi_filter_find_vlan(struct efx_nic * efx,u16 vid)1653 struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx,
1654 u16 vid)
1655 {
1656 struct efx_mcdi_filter_table *table = efx->filter_state;
1657 struct efx_mcdi_filter_vlan *vlan;
1658
1659 WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1660
1661 list_for_each_entry(vlan, &table->vlan_list, list) {
1662 if (vlan->vid == vid)
1663 return vlan;
1664 }
1665
1666 return NULL;
1667 }
1668
efx_mcdi_filter_cleanup_vlans(struct efx_nic * efx)1669 void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx)
1670 {
1671 struct efx_mcdi_filter_table *table = efx->filter_state;
1672 struct efx_mcdi_filter_vlan *vlan, *next_vlan;
1673
1674 /* See comment in efx_mcdi_filter_table_remove() */
1675 if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1676 return;
1677
1678 if (!table)
1679 return;
1680
1681 list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
1682 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1683 }
1684
efx_mcdi_filter_uc_addr_list(struct efx_nic * efx)1685 static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx)
1686 {
1687 struct efx_mcdi_filter_table *table = efx->filter_state;
1688 struct net_device *net_dev = efx->net_dev;
1689 struct netdev_hw_addr *uc;
1690 unsigned int i;
1691
1692 table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
1693 ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
1694 i = 1;
1695 netdev_for_each_uc_addr(uc, net_dev) {
1696 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
1697 table->uc_promisc = true;
1698 break;
1699 }
1700 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
1701 i++;
1702 }
1703
1704 table->dev_uc_count = i;
1705 }
1706
efx_mcdi_filter_mc_addr_list(struct efx_nic * efx)1707 static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx)
1708 {
1709 struct efx_mcdi_filter_table *table = efx->filter_state;
1710 struct net_device *net_dev = efx->net_dev;
1711 struct netdev_hw_addr *mc;
1712 unsigned int i;
1713
1714 table->mc_overflow = false;
1715 table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));
1716
1717 i = 0;
1718 netdev_for_each_mc_addr(mc, net_dev) {
1719 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
1720 table->mc_promisc = true;
1721 table->mc_overflow = true;
1722 break;
1723 }
1724 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
1725 i++;
1726 }
1727
1728 table->dev_mc_count = i;
1729 }
1730
1731 /*
1732 * Caller must hold efx->filter_sem for read if race against
1733 * efx_mcdi_filter_table_remove() is possible
1734 */
efx_mcdi_filter_sync_rx_mode(struct efx_nic * efx)1735 void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx)
1736 {
1737 struct efx_mcdi_filter_table *table = efx->filter_state;
1738 struct net_device *net_dev = efx->net_dev;
1739 struct efx_mcdi_filter_vlan *vlan;
1740 bool vlan_filter;
1741
1742 if (!efx_dev_registered(efx))
1743 return;
1744
1745 if (!table)
1746 return;
1747
1748 efx_mcdi_filter_mark_old(efx);
1749
1750 /*
1751 * Copy/convert the address lists; add the primary station
1752 * address and broadcast address
1753 */
1754 netif_addr_lock_bh(net_dev);
1755 efx_mcdi_filter_uc_addr_list(efx);
1756 efx_mcdi_filter_mc_addr_list(efx);
1757 netif_addr_unlock_bh(net_dev);
1758
1759 /*
1760 * If VLAN filtering changes, all old filters are finally removed.
1761 * Do it in advance to avoid conflicts for unicast untagged and
1762 * VLAN 0 tagged filters.
1763 */
1764 vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1765 if (table->vlan_filter != vlan_filter) {
1766 table->vlan_filter = vlan_filter;
1767 efx_mcdi_filter_remove_old(efx);
1768 }
1769
1770 list_for_each_entry(vlan, &table->vlan_list, list)
1771 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1772
1773 efx_mcdi_filter_remove_old(efx);
1774 table->mc_promisc_last = table->mc_promisc;
1775 }
1776
1777 #ifdef CONFIG_RFS_ACCEL
1778
efx_mcdi_filter_rfs_expire_one(struct efx_nic * efx,u32 flow_id,unsigned int filter_idx)1779 bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
1780 unsigned int filter_idx)
1781 {
1782 struct efx_filter_spec *spec, saved_spec;
1783 struct efx_mcdi_filter_table *table;
1784 struct efx_arfs_rule *rule = NULL;
1785 bool ret = true, force = false;
1786 u16 arfs_id;
1787
1788 down_read(&efx->filter_sem);
1789 table = efx->filter_state;
1790 down_write(&table->lock);
1791 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1792
1793 if (!spec || spec->priority != EFX_FILTER_PRI_HINT)
1794 goto out_unlock;
1795
1796 spin_lock_bh(&efx->rps_hash_lock);
1797 if (!efx->rps_hash_table) {
1798 /* In the absence of the table, we always return 0 to ARFS. */
1799 arfs_id = 0;
1800 } else {
1801 rule = efx_rps_hash_find(efx, spec);
1802 if (!rule)
1803 /* ARFS table doesn't know of this filter, so remove it */
1804 goto expire;
1805 arfs_id = rule->arfs_id;
1806 ret = efx_rps_check_rule(rule, filter_idx, &force);
1807 if (force)
1808 goto expire;
1809 if (!ret) {
1810 spin_unlock_bh(&efx->rps_hash_lock);
1811 goto out_unlock;
1812 }
1813 }
1814 if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))
1815 ret = false;
1816 else if (rule)
1817 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
1818 expire:
1819 saved_spec = *spec; /* remove operation will kfree spec */
1820 spin_unlock_bh(&efx->rps_hash_lock);
1821 /*
1822 * At this point (since we dropped the lock), another thread might queue
1823 * up a fresh insertion request (but the actual insertion will be held
1824 * up by our possession of the filter table lock). In that case, it
1825 * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that
1826 * the rule is not removed by efx_rps_hash_del() below.
1827 */
1828 if (ret)
1829 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority,
1830 filter_idx, true) == 0;
1831 /*
1832 * While we can't safely dereference rule (we dropped the lock), we can
1833 * still test it for NULL.
1834 */
1835 if (ret && rule) {
1836 /* Expiring, so remove entry from ARFS table */
1837 spin_lock_bh(&efx->rps_hash_lock);
1838 efx_rps_hash_del(efx, &saved_spec);
1839 spin_unlock_bh(&efx->rps_hash_lock);
1840 }
1841 out_unlock:
1842 up_write(&table->lock);
1843 up_read(&efx->filter_sem);
1844 return ret;
1845 }
1846
1847 #endif /* CONFIG_RFS_ACCEL */
1848
1849 #define RSS_MODE_HASH_ADDRS (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\
1850 1 << RSS_MODE_HASH_DST_ADDR_LBN)
1851 #define RSS_MODE_HASH_PORTS (1 << RSS_MODE_HASH_SRC_PORT_LBN |\
1852 1 << RSS_MODE_HASH_DST_PORT_LBN)
1853 #define RSS_CONTEXT_FLAGS_DEFAULT (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\
1854 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\
1855 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\
1856 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\
1857 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\
1858 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\
1859 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\
1860 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\
1861 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\
1862 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN)
1863
efx_mcdi_get_rss_context_flags(struct efx_nic * efx,u32 context,u32 * flags)1864 int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags)
1865 {
1866 /*
1867 * Firmware had a bug (sfc bug 61952) where it would not actually
1868 * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS.
1869 * This meant that it would always contain whatever was previously
1870 * in the MCDI buffer. Fortunately, all firmware versions with
1871 * this bug have the same default flags value for a newly-allocated
1872 * RSS context, and the only time we want to get the flags is just
1873 * after allocating. Moreover, the response has a 32-bit hole
1874 * where the context ID would be in the request, so we can use an
1875 * overlength buffer in the request and pre-fill the flags field
1876 * with what we believe the default to be. Thus if the firmware
1877 * has the bug, it will leave our pre-filled value in the flags
1878 * field of the response, and we will get the right answer.
1879 *
1880 * However, this does mean that this function should NOT be used if
1881 * the RSS context flags might not be their defaults - it is ONLY
1882 * reliably correct for a newly-allocated RSS context.
1883 */
1884 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1885 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1886 size_t outlen;
1887 int rc;
1888
1889 /* Check we have a hole for the context ID */
1890 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST);
1891 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context);
1892 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS,
1893 RSS_CONTEXT_FLAGS_DEFAULT);
1894 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf,
1895 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
1896 if (rc == 0) {
1897 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN)
1898 rc = -EIO;
1899 else
1900 *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS);
1901 }
1902 return rc;
1903 }
1904
1905 /*
1906 * Attempt to enable 4-tuple UDP hashing on the specified RSS context.
1907 * If we fail, we just leave the RSS context at its default hash settings,
1908 * which is safe but may slightly reduce performance.
1909 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we
1910 * just need to set the UDP ports flags (for both IP versions).
1911 */
efx_mcdi_set_rss_context_flags(struct efx_nic * efx,struct efx_rss_context * ctx)1912 void efx_mcdi_set_rss_context_flags(struct efx_nic *efx,
1913 struct efx_rss_context *ctx)
1914 {
1915 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN);
1916 u32 flags;
1917
1918 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0);
1919
1920 if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0)
1921 return;
1922 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
1923 ctx->context_id);
1924 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN;
1925 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN;
1926 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags);
1927 if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf),
1928 NULL, 0, NULL))
1929 /* Succeeded, so UDP 4-tuple is now enabled */
1930 ctx->rx_hash_udp_4tuple = true;
1931 }
1932
efx_mcdi_filter_alloc_rss_context(struct efx_nic * efx,bool exclusive,struct efx_rss_context * ctx,unsigned * context_size)1933 static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive,
1934 struct efx_rss_context *ctx,
1935 unsigned *context_size)
1936 {
1937 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
1938 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
1939 size_t outlen;
1940 int rc;
1941 u32 alloc_type = exclusive ?
1942 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE :
1943 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
1944 unsigned rss_spread = exclusive ?
1945 efx->rss_spread :
1946 min(rounddown_pow_of_two(efx->rss_spread),
1947 EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE);
1948
1949 if (!exclusive && rss_spread == 1) {
1950 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
1951 if (context_size)
1952 *context_size = 1;
1953 return 0;
1954 }
1955
1956 if (efx_has_cap(efx, RX_RSS_LIMITED))
1957 return -EOPNOTSUPP;
1958
1959 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
1960 efx->vport_id);
1961 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type);
1962 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread);
1963
1964 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
1965 outbuf, sizeof(outbuf), &outlen);
1966 if (rc != 0)
1967 return rc;
1968
1969 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
1970 return -EIO;
1971
1972 ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
1973
1974 if (context_size)
1975 *context_size = rss_spread;
1976
1977 if (efx_has_cap(efx, ADDITIONAL_RSS_MODES))
1978 efx_mcdi_set_rss_context_flags(efx, ctx);
1979
1980 return 0;
1981 }
1982
efx_mcdi_filter_free_rss_context(struct efx_nic * efx,u32 context)1983 static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context)
1984 {
1985 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);
1986
1987 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
1988 context);
1989 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
1990 NULL, 0, NULL);
1991 }
1992
efx_mcdi_filter_populate_rss_table(struct efx_nic * efx,u32 context,const u32 * rx_indir_table,const u8 * key)1993 static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context,
1994 const u32 *rx_indir_table, const u8 *key)
1995 {
1996 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
1997 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
1998 int i, rc;
1999
2000 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
2001 context);
2002 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
2003 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);
2004
2005 /* This iterates over the length of efx->rss_context.rx_indir_table, but
2006 * copies bytes from rx_indir_table. That's because the latter is a
2007 * pointer rather than an array, but should have the same length.
2008 * The efx->rss_context.rx_hash_key loop below is similar.
2009 */
2010 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i)
2011 MCDI_PTR(tablebuf,
2012 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
2013 (u8) rx_indir_table[i];
2014
2015 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
2016 sizeof(tablebuf), NULL, 0, NULL);
2017 if (rc != 0)
2018 return rc;
2019
2020 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
2021 context);
2022 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) !=
2023 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2024 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i)
2025 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i];
2026
2027 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
2028 sizeof(keybuf), NULL, 0, NULL);
2029 }
2030
efx_mcdi_rx_free_indir_table(struct efx_nic * efx)2031 void efx_mcdi_rx_free_indir_table(struct efx_nic *efx)
2032 {
2033 int rc;
2034
2035 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) {
2036 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id);
2037 WARN_ON(rc != 0);
2038 }
2039 efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2040 }
2041
efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic * efx,unsigned * context_size)2042 static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx,
2043 unsigned *context_size)
2044 {
2045 struct efx_mcdi_filter_table *table = efx->filter_state;
2046 int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context,
2047 context_size);
2048
2049 if (rc != 0)
2050 return rc;
2051
2052 table->rx_rss_context_exclusive = false;
2053 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2054 return 0;
2055 }
2056
efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic * efx,const u32 * rx_indir_table,const u8 * key)2057 static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx,
2058 const u32 *rx_indir_table,
2059 const u8 *key)
2060 {
2061 struct efx_mcdi_filter_table *table = efx->filter_state;
2062 u32 old_rx_rss_context = efx->rss_context.context_id;
2063 int rc;
2064
2065 if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID ||
2066 !table->rx_rss_context_exclusive) {
2067 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context,
2068 NULL);
2069 if (rc == -EOPNOTSUPP)
2070 return rc;
2071 else if (rc != 0)
2072 goto fail1;
2073 }
2074
2075 rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id,
2076 rx_indir_table, key);
2077 if (rc != 0)
2078 goto fail2;
2079
2080 if (efx->rss_context.context_id != old_rx_rss_context &&
2081 old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID)
2082 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0);
2083 table->rx_rss_context_exclusive = true;
2084 if (rx_indir_table != efx->rss_context.rx_indir_table)
2085 memcpy(efx->rss_context.rx_indir_table, rx_indir_table,
2086 sizeof(efx->rss_context.rx_indir_table));
2087 if (key != efx->rss_context.rx_hash_key)
2088 memcpy(efx->rss_context.rx_hash_key, key,
2089 efx->type->rx_hash_key_size);
2090
2091 return 0;
2092
2093 fail2:
2094 if (old_rx_rss_context != efx->rss_context.context_id) {
2095 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0);
2096 efx->rss_context.context_id = old_rx_rss_context;
2097 }
2098 fail1:
2099 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
2100 return rc;
2101 }
2102
efx_mcdi_rx_push_rss_context_config(struct efx_nic * efx,struct efx_rss_context * ctx,const u32 * rx_indir_table,const u8 * key)2103 int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx,
2104 struct efx_rss_context *ctx,
2105 const u32 *rx_indir_table,
2106 const u8 *key)
2107 {
2108 int rc;
2109
2110 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2111
2112 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
2113 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL);
2114 if (rc)
2115 return rc;
2116 }
2117
2118 if (!rx_indir_table) /* Delete this context */
2119 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id);
2120
2121 rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id,
2122 rx_indir_table, key);
2123 if (rc)
2124 return rc;
2125
2126 memcpy(ctx->rx_indir_table, rx_indir_table,
2127 sizeof(efx->rss_context.rx_indir_table));
2128 memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size);
2129
2130 return 0;
2131 }
2132
efx_mcdi_rx_pull_rss_context_config(struct efx_nic * efx,struct efx_rss_context * ctx)2133 int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx,
2134 struct efx_rss_context *ctx)
2135 {
2136 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN);
2137 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN);
2138 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN);
2139 size_t outlen;
2140 int rc, i;
2141
2142 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2143
2144 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
2145 MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
2146
2147 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)
2148 return -ENOENT;
2149
2150 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID,
2151 ctx->context_id);
2152 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) !=
2153 MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN);
2154 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf),
2155 tablebuf, sizeof(tablebuf), &outlen);
2156 if (rc != 0)
2157 return rc;
2158
2159 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN))
2160 return -EIO;
2161
2162 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
2163 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf,
2164 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i];
2165
2166 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID,
2167 ctx->context_id);
2168 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) !=
2169 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2170 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf),
2171 keybuf, sizeof(keybuf), &outlen);
2172 if (rc != 0)
2173 return rc;
2174
2175 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN))
2176 return -EIO;
2177
2178 for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i)
2179 ctx->rx_hash_key[i] = MCDI_PTR(
2180 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i];
2181
2182 return 0;
2183 }
2184
efx_mcdi_rx_pull_rss_config(struct efx_nic * efx)2185 int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx)
2186 {
2187 int rc;
2188
2189 mutex_lock(&efx->rss_lock);
2190 rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context);
2191 mutex_unlock(&efx->rss_lock);
2192 return rc;
2193 }
2194
efx_mcdi_rx_restore_rss_contexts(struct efx_nic * efx)2195 void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx)
2196 {
2197 struct efx_mcdi_filter_table *table = efx->filter_state;
2198 struct efx_rss_context *ctx;
2199 int rc;
2200
2201 WARN_ON(!mutex_is_locked(&efx->rss_lock));
2202
2203 if (!table->must_restore_rss_contexts)
2204 return;
2205
2206 list_for_each_entry(ctx, &efx->rss_context.list, list) {
2207 /* previous NIC RSS context is gone */
2208 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2209 /* so try to allocate a new one */
2210 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx,
2211 ctx->rx_indir_table,
2212 ctx->rx_hash_key);
2213 if (rc)
2214 netif_warn(efx, probe, efx->net_dev,
2215 "failed to restore RSS context %u, rc=%d"
2216 "; RSS filters may fail to be applied\n",
2217 ctx->user_id, rc);
2218 }
2219 table->must_restore_rss_contexts = false;
2220 }
2221
efx_mcdi_pf_rx_push_rss_config(struct efx_nic * efx,bool user,const u32 * rx_indir_table,const u8 * key)2222 int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
2223 const u32 *rx_indir_table,
2224 const u8 *key)
2225 {
2226 int rc;
2227
2228 if (efx->rss_spread == 1)
2229 return 0;
2230
2231 if (!key)
2232 key = efx->rss_context.rx_hash_key;
2233
2234 rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key);
2235
2236 if (rc == -ENOBUFS && !user) {
2237 unsigned context_size;
2238 bool mismatch = false;
2239 size_t i;
2240
2241 for (i = 0;
2242 i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch;
2243 i++)
2244 mismatch = rx_indir_table[i] !=
2245 ethtool_rxfh_indir_default(i, efx->rss_spread);
2246
2247 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size);
2248 if (rc == 0) {
2249 if (context_size != efx->rss_spread)
2250 netif_warn(efx, probe, efx->net_dev,
2251 "Could not allocate an exclusive RSS"
2252 " context; allocated a shared one of"
2253 " different size."
2254 " Wanted %u, got %u.\n",
2255 efx->rss_spread, context_size);
2256 else if (mismatch)
2257 netif_warn(efx, probe, efx->net_dev,
2258 "Could not allocate an exclusive RSS"
2259 " context; allocated a shared one but"
2260 " could not apply custom"
2261 " indirection.\n");
2262 else
2263 netif_info(efx, probe, efx->net_dev,
2264 "Could not allocate an exclusive RSS"
2265 " context; allocated a shared one.\n");
2266 }
2267 }
2268 return rc;
2269 }
2270
efx_mcdi_vf_rx_push_rss_config(struct efx_nic * efx,bool user,const u32 * rx_indir_table,const u8 * key)2271 int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user,
2272 const u32 *rx_indir_table
2273 __attribute__ ((unused)),
2274 const u8 *key
2275 __attribute__ ((unused)))
2276 {
2277 if (user)
2278 return -EOPNOTSUPP;
2279 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID)
2280 return 0;
2281 return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL);
2282 }
2283
efx_mcdi_push_default_indir_table(struct efx_nic * efx,unsigned int rss_spread)2284 int efx_mcdi_push_default_indir_table(struct efx_nic *efx,
2285 unsigned int rss_spread)
2286 {
2287 int rc = 0;
2288
2289 if (efx->rss_spread == rss_spread)
2290 return 0;
2291
2292 efx->rss_spread = rss_spread;
2293 if (!efx->filter_state)
2294 return 0;
2295
2296 efx_mcdi_rx_free_indir_table(efx);
2297 if (rss_spread > 1) {
2298 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2299 rc = efx->type->rx_push_rss_config(efx, false,
2300 efx->rss_context.rx_indir_table, NULL);
2301 }
2302 return rc;
2303 }
2304