1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies. */
3
4 #include <linux/refcount.h>
5 #include <linux/list.h>
6 #include <linux/rculist.h>
7 #include <linux/rtnetlink.h>
8 #include <linux/workqueue.h>
9 #include <linux/spinlock.h>
10 #include <linux/notifier.h>
11 #include <net/netevent.h>
12 #include <net/arp.h>
13 #include "neigh.h"
14 #include "tc.h"
15 #include "en_rep.h"
16 #include "fs_core.h"
17 #include "diag/en_rep_tracepoint.h"
18
mlx5e_rep_ipv6_interval(void)19 static unsigned long mlx5e_rep_ipv6_interval(void)
20 {
21 if (IS_ENABLED(CONFIG_IPV6) && ipv6_stub->nd_tbl)
22 return NEIGH_VAR(&ipv6_stub->nd_tbl->parms, DELAY_PROBE_TIME);
23
24 return ~0UL;
25 }
26
mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv * rpriv)27 static void mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv *rpriv)
28 {
29 unsigned long ipv4_interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
30 unsigned long ipv6_interval = mlx5e_rep_ipv6_interval();
31 struct net_device *netdev = rpriv->netdev;
32 struct mlx5e_priv *priv = netdev_priv(netdev);
33
34 rpriv->neigh_update.min_interval = min_t(unsigned long, ipv6_interval, ipv4_interval);
35 mlx5_fc_update_sampling_interval(priv->mdev, rpriv->neigh_update.min_interval);
36 }
37
mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv * priv)38 void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv)
39 {
40 struct mlx5e_rep_priv *rpriv = priv->ppriv;
41 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
42
43 mlx5_fc_queue_stats_work(priv->mdev,
44 &neigh_update->neigh_stats_work,
45 neigh_update->min_interval);
46 }
47
mlx5e_rep_neigh_entry_hold(struct mlx5e_neigh_hash_entry * nhe)48 static bool mlx5e_rep_neigh_entry_hold(struct mlx5e_neigh_hash_entry *nhe)
49 {
50 return refcount_inc_not_zero(&nhe->refcnt);
51 }
52
53 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe);
54
mlx5e_rep_neigh_entry_release(struct mlx5e_neigh_hash_entry * nhe)55 void mlx5e_rep_neigh_entry_release(struct mlx5e_neigh_hash_entry *nhe)
56 {
57 if (refcount_dec_and_test(&nhe->refcnt)) {
58 mlx5e_rep_neigh_entry_remove(nhe);
59 kfree_rcu(nhe, rcu);
60 }
61 }
62
63 static struct mlx5e_neigh_hash_entry *
mlx5e_get_next_nhe(struct mlx5e_rep_priv * rpriv,struct mlx5e_neigh_hash_entry * nhe)64 mlx5e_get_next_nhe(struct mlx5e_rep_priv *rpriv,
65 struct mlx5e_neigh_hash_entry *nhe)
66 {
67 struct mlx5e_neigh_hash_entry *next = NULL;
68
69 rcu_read_lock();
70
71 for (next = nhe ?
72 list_next_or_null_rcu(&rpriv->neigh_update.neigh_list,
73 &nhe->neigh_list,
74 struct mlx5e_neigh_hash_entry,
75 neigh_list) :
76 list_first_or_null_rcu(&rpriv->neigh_update.neigh_list,
77 struct mlx5e_neigh_hash_entry,
78 neigh_list);
79 next;
80 next = list_next_or_null_rcu(&rpriv->neigh_update.neigh_list,
81 &next->neigh_list,
82 struct mlx5e_neigh_hash_entry,
83 neigh_list))
84 if (mlx5e_rep_neigh_entry_hold(next))
85 break;
86
87 rcu_read_unlock();
88
89 if (nhe)
90 mlx5e_rep_neigh_entry_release(nhe);
91
92 return next;
93 }
94
mlx5e_rep_neigh_stats_work(struct work_struct * work)95 static void mlx5e_rep_neigh_stats_work(struct work_struct *work)
96 {
97 struct mlx5e_rep_priv *rpriv = container_of(work, struct mlx5e_rep_priv,
98 neigh_update.neigh_stats_work.work);
99 struct net_device *netdev = rpriv->netdev;
100 struct mlx5e_priv *priv = netdev_priv(netdev);
101 struct mlx5e_neigh_hash_entry *nhe = NULL;
102
103 rtnl_lock();
104 if (!list_empty(&rpriv->neigh_update.neigh_list))
105 mlx5e_rep_queue_neigh_stats_work(priv);
106
107 while ((nhe = mlx5e_get_next_nhe(rpriv, nhe)) != NULL)
108 mlx5e_tc_update_neigh_used_value(nhe);
109
110 rtnl_unlock();
111 }
112
113 struct neigh_update_work {
114 struct work_struct work;
115 struct neighbour *n;
116 struct mlx5e_neigh_hash_entry *nhe;
117 };
118
mlx5e_release_neigh_update_work(struct neigh_update_work * update_work)119 static void mlx5e_release_neigh_update_work(struct neigh_update_work *update_work)
120 {
121 neigh_release(update_work->n);
122 mlx5e_rep_neigh_entry_release(update_work->nhe);
123 kfree(update_work);
124 }
125
mlx5e_rep_neigh_update(struct work_struct * work)126 static void mlx5e_rep_neigh_update(struct work_struct *work)
127 {
128 struct neigh_update_work *update_work = container_of(work, struct neigh_update_work,
129 work);
130 struct mlx5e_neigh_hash_entry *nhe = update_work->nhe;
131 struct neighbour *n = update_work->n;
132 struct mlx5e_encap_entry *e;
133 unsigned char ha[ETH_ALEN];
134 struct mlx5e_priv *priv;
135 bool neigh_connected;
136 u8 nud_state, dead;
137
138 rtnl_lock();
139
140 /* If these parameters are changed after we release the lock,
141 * we'll receive another event letting us know about it.
142 * We use this lock to avoid inconsistency between the neigh validity
143 * and it's hw address.
144 */
145 read_lock_bh(&n->lock);
146 memcpy(ha, n->ha, ETH_ALEN);
147 nud_state = n->nud_state;
148 dead = n->dead;
149 read_unlock_bh(&n->lock);
150
151 neigh_connected = (nud_state & NUD_VALID) && !dead;
152
153 trace_mlx5e_rep_neigh_update(nhe, ha, neigh_connected);
154
155 list_for_each_entry(e, &nhe->encap_list, encap_list) {
156 if (!mlx5e_encap_take(e))
157 continue;
158
159 priv = netdev_priv(e->out_dev);
160 mlx5e_rep_update_flows(priv, e, neigh_connected, ha);
161 mlx5e_encap_put(priv, e);
162 }
163 rtnl_unlock();
164 mlx5e_release_neigh_update_work(update_work);
165 }
166
mlx5e_alloc_neigh_update_work(struct mlx5e_priv * priv,struct neighbour * n)167 static struct neigh_update_work *mlx5e_alloc_neigh_update_work(struct mlx5e_priv *priv,
168 struct neighbour *n)
169 {
170 struct neigh_update_work *update_work;
171 struct mlx5e_neigh_hash_entry *nhe;
172 struct mlx5e_neigh m_neigh = {};
173
174 update_work = kzalloc(sizeof(*update_work), GFP_ATOMIC);
175 if (WARN_ON(!update_work))
176 return NULL;
177
178 m_neigh.dev = n->dev;
179 m_neigh.family = n->ops->family;
180 memcpy(&m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
181
182 /* Obtain reference to nhe as last step in order not to release it in
183 * atomic context.
184 */
185 rcu_read_lock();
186 nhe = mlx5e_rep_neigh_entry_lookup(priv, &m_neigh);
187 rcu_read_unlock();
188 if (!nhe) {
189 kfree(update_work);
190 return NULL;
191 }
192
193 INIT_WORK(&update_work->work, mlx5e_rep_neigh_update);
194 neigh_hold(n);
195 update_work->n = n;
196 update_work->nhe = nhe;
197
198 return update_work;
199 }
200
mlx5e_rep_netevent_event(struct notifier_block * nb,unsigned long event,void * ptr)201 static int mlx5e_rep_netevent_event(struct notifier_block *nb,
202 unsigned long event, void *ptr)
203 {
204 struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv,
205 neigh_update.netevent_nb);
206 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
207 struct net_device *netdev = rpriv->netdev;
208 struct mlx5e_priv *priv = netdev_priv(netdev);
209 struct mlx5e_neigh_hash_entry *nhe = NULL;
210 struct neigh_update_work *update_work;
211 struct neigh_parms *p;
212 struct neighbour *n;
213 bool found = false;
214
215 switch (event) {
216 case NETEVENT_NEIGH_UPDATE:
217 n = ptr;
218 #if IS_ENABLED(CONFIG_IPV6)
219 if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl)
220 #else
221 if (n->tbl != &arp_tbl)
222 #endif
223 return NOTIFY_DONE;
224
225 update_work = mlx5e_alloc_neigh_update_work(priv, n);
226 if (!update_work)
227 return NOTIFY_DONE;
228
229 queue_work(priv->wq, &update_work->work);
230 break;
231
232 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
233 p = ptr;
234
235 /* We check the device is present since we don't care about
236 * changes in the default table, we only care about changes
237 * done per device delay prob time parameter.
238 */
239 #if IS_ENABLED(CONFIG_IPV6)
240 if (!p->dev || (p->tbl != ipv6_stub->nd_tbl && p->tbl != &arp_tbl))
241 #else
242 if (!p->dev || p->tbl != &arp_tbl)
243 #endif
244 return NOTIFY_DONE;
245
246 rcu_read_lock();
247 list_for_each_entry_rcu(nhe, &neigh_update->neigh_list,
248 neigh_list) {
249 if (p->dev == nhe->m_neigh.dev) {
250 found = true;
251 break;
252 }
253 }
254 rcu_read_unlock();
255 if (!found)
256 return NOTIFY_DONE;
257
258 neigh_update->min_interval = min_t(unsigned long,
259 NEIGH_VAR(p, DELAY_PROBE_TIME),
260 neigh_update->min_interval);
261 mlx5_fc_update_sampling_interval(priv->mdev,
262 neigh_update->min_interval);
263 break;
264 }
265 return NOTIFY_DONE;
266 }
267
268 static const struct rhashtable_params mlx5e_neigh_ht_params = {
269 .head_offset = offsetof(struct mlx5e_neigh_hash_entry, rhash_node),
270 .key_offset = offsetof(struct mlx5e_neigh_hash_entry, m_neigh),
271 .key_len = sizeof(struct mlx5e_neigh),
272 .automatic_shrinking = true,
273 };
274
mlx5e_rep_neigh_init(struct mlx5e_rep_priv * rpriv)275 int mlx5e_rep_neigh_init(struct mlx5e_rep_priv *rpriv)
276 {
277 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
278 int err;
279
280 err = rhashtable_init(&neigh_update->neigh_ht, &mlx5e_neigh_ht_params);
281 if (err)
282 return err;
283
284 INIT_LIST_HEAD(&neigh_update->neigh_list);
285 mutex_init(&neigh_update->encap_lock);
286 INIT_DELAYED_WORK(&neigh_update->neigh_stats_work,
287 mlx5e_rep_neigh_stats_work);
288 mlx5e_rep_neigh_update_init_interval(rpriv);
289
290 rpriv->neigh_update.netevent_nb.notifier_call = mlx5e_rep_netevent_event;
291 err = register_netevent_notifier(&rpriv->neigh_update.netevent_nb);
292 if (err)
293 goto out_err;
294 return 0;
295
296 out_err:
297 rhashtable_destroy(&neigh_update->neigh_ht);
298 return err;
299 }
300
mlx5e_rep_neigh_cleanup(struct mlx5e_rep_priv * rpriv)301 void mlx5e_rep_neigh_cleanup(struct mlx5e_rep_priv *rpriv)
302 {
303 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
304 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
305
306 unregister_netevent_notifier(&neigh_update->netevent_nb);
307
308 flush_workqueue(priv->wq); /* flush neigh update works */
309
310 cancel_delayed_work_sync(&rpriv->neigh_update.neigh_stats_work);
311
312 mutex_destroy(&neigh_update->encap_lock);
313 rhashtable_destroy(&neigh_update->neigh_ht);
314 }
315
mlx5e_rep_neigh_entry_insert(struct mlx5e_priv * priv,struct mlx5e_neigh_hash_entry * nhe)316 static int mlx5e_rep_neigh_entry_insert(struct mlx5e_priv *priv,
317 struct mlx5e_neigh_hash_entry *nhe)
318 {
319 struct mlx5e_rep_priv *rpriv = priv->ppriv;
320 int err;
321
322 err = rhashtable_insert_fast(&rpriv->neigh_update.neigh_ht,
323 &nhe->rhash_node,
324 mlx5e_neigh_ht_params);
325 if (err)
326 return err;
327
328 list_add_rcu(&nhe->neigh_list, &rpriv->neigh_update.neigh_list);
329
330 return err;
331 }
332
mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry * nhe)333 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe)
334 {
335 struct mlx5e_rep_priv *rpriv = nhe->priv->ppriv;
336
337 mutex_lock(&rpriv->neigh_update.encap_lock);
338
339 list_del_rcu(&nhe->neigh_list);
340
341 rhashtable_remove_fast(&rpriv->neigh_update.neigh_ht,
342 &nhe->rhash_node,
343 mlx5e_neigh_ht_params);
344 mutex_unlock(&rpriv->neigh_update.encap_lock);
345 }
346
347 /* This function must only be called under the representor's encap_lock or
348 * inside rcu read lock section.
349 */
350 struct mlx5e_neigh_hash_entry *
mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv * priv,struct mlx5e_neigh * m_neigh)351 mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv *priv,
352 struct mlx5e_neigh *m_neigh)
353 {
354 struct mlx5e_rep_priv *rpriv = priv->ppriv;
355 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
356 struct mlx5e_neigh_hash_entry *nhe;
357
358 nhe = rhashtable_lookup_fast(&neigh_update->neigh_ht, m_neigh,
359 mlx5e_neigh_ht_params);
360 return nhe && mlx5e_rep_neigh_entry_hold(nhe) ? nhe : NULL;
361 }
362
mlx5e_rep_neigh_entry_create(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct mlx5e_neigh_hash_entry ** nhe)363 int mlx5e_rep_neigh_entry_create(struct mlx5e_priv *priv,
364 struct mlx5e_encap_entry *e,
365 struct mlx5e_neigh_hash_entry **nhe)
366 {
367 int err;
368
369 *nhe = kzalloc(sizeof(**nhe), GFP_KERNEL);
370 if (!*nhe)
371 return -ENOMEM;
372
373 (*nhe)->priv = priv;
374 memcpy(&(*nhe)->m_neigh, &e->m_neigh, sizeof(e->m_neigh));
375 spin_lock_init(&(*nhe)->encap_list_lock);
376 INIT_LIST_HEAD(&(*nhe)->encap_list);
377 refcount_set(&(*nhe)->refcnt, 1);
378
379 err = mlx5e_rep_neigh_entry_insert(priv, *nhe);
380 if (err)
381 goto out_free;
382 return 0;
383
384 out_free:
385 kfree(*nhe);
386 return err;
387 }
388