1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2018 Mellanox Technologies. */
3
4 #include <net/flow.h>
5 #include <net/inet_dscp.h>
6 #include <net/vxlan.h>
7 #include <net/gre.h>
8 #include <net/geneve.h>
9 #include <net/bareudp.h>
10 #include "en/tc_tun.h"
11 #include "en/tc_priv.h"
12 #include "en_tc.h"
13 #include "rep/tc.h"
14 #include "rep/neigh.h"
15 #include "lag/lag.h"
16 #include "lag/mp.h"
17
18 struct mlx5e_tc_tun_route_attr {
19 struct net_device *out_dev;
20 struct net_device *route_dev;
21 union {
22 struct flowi4 fl4;
23 struct flowi6 fl6;
24 } fl;
25 struct neighbour *n;
26 u8 ttl;
27 };
28
29 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {}
30
mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr * attr)31 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr)
32 {
33 if (attr->n)
34 neigh_release(attr->n);
35 dev_put(attr->route_dev);
36 }
37
mlx5e_get_tc_tun(struct net_device * tunnel_dev)38 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev)
39 {
40 if (netif_is_vxlan(tunnel_dev))
41 return &vxlan_tunnel;
42 else if (netif_is_geneve(tunnel_dev))
43 return &geneve_tunnel;
44 else if (netif_is_gretap(tunnel_dev) ||
45 netif_is_ip6gretap(tunnel_dev))
46 return &gre_tunnel;
47 else if (netif_is_bareudp(tunnel_dev))
48 return &mplsoudp_tunnel;
49 else
50 return NULL;
51 }
52
get_route_and_out_devs(struct mlx5e_priv * priv,struct net_device * dev,struct net_device ** route_dev,struct net_device ** out_dev)53 static int get_route_and_out_devs(struct mlx5e_priv *priv,
54 struct net_device *dev,
55 struct net_device **route_dev,
56 struct net_device **out_dev)
57 {
58 struct net_device *uplink_dev, *uplink_upper, *real_dev;
59 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
60 bool dst_is_lag_dev;
61
62 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev;
63 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
64
65 rcu_read_lock();
66 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev);
67 /* mlx5_lag_is_sriov() is a blocking function which can't be called
68 * while holding rcu read lock. Take the net_device for correctness
69 * sake.
70 */
71 dev_hold(uplink_upper);
72 rcu_read_unlock();
73
74 dst_is_lag_dev = (uplink_upper &&
75 netif_is_lag_master(uplink_upper) &&
76 real_dev == uplink_upper &&
77 mlx5_lag_is_sriov(priv->mdev));
78 dev_put(uplink_upper);
79
80 /* if the egress device isn't on the same HW e-switch or
81 * it's a LAG device, use the uplink
82 */
83 *route_dev = dev;
84 if (!netdev_port_same_parent_id(priv->netdev, real_dev) ||
85 dst_is_lag_dev || is_vlan_dev(*route_dev) ||
86 netif_is_ovs_master(*route_dev))
87 *out_dev = uplink_dev;
88 else if (mlx5e_eswitch_rep(dev) &&
89 mlx5e_is_valid_eswitch_fwd_dev(priv, dev))
90 *out_dev = *route_dev;
91 else
92 return -EOPNOTSUPP;
93
94 if (!mlx5e_eswitch_uplink_rep(*out_dev))
95 return -EOPNOTSUPP;
96
97 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev &&
98 !mlx5_lag_is_mpesw(priv->mdev))
99 return -EOPNOTSUPP;
100
101 return 0;
102 }
103
mlx5e_route_lookup_ipv4_get(struct mlx5e_priv * priv,struct net_device * dev,struct mlx5e_tc_tun_route_attr * attr)104 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv,
105 struct net_device *dev,
106 struct mlx5e_tc_tun_route_attr *attr)
107 {
108 struct net_device *route_dev;
109 struct net_device *out_dev;
110 struct neighbour *n;
111 struct rtable *rt;
112
113 #if IS_ENABLED(CONFIG_INET)
114 struct mlx5_core_dev *mdev = priv->mdev;
115 struct net_device *uplink_dev;
116 int ret;
117
118 if (mlx5_lag_is_multipath(mdev)) {
119 struct mlx5_eswitch *esw = mdev->priv.eswitch;
120
121 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
122 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex;
123 } else {
124 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
125
126 if (tunnel && tunnel->get_remote_ifindex)
127 attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(dev);
128 }
129
130 rt = ip_route_output_key(dev_net(dev), &attr->fl.fl4);
131 if (IS_ERR(rt))
132 return PTR_ERR(rt);
133
134 if (rt->rt_type != RTN_UNICAST) {
135 ret = -ENETUNREACH;
136 goto err_rt_release;
137 }
138
139 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) {
140 ret = -ENETUNREACH;
141 goto err_rt_release;
142 }
143 #else
144 return -EOPNOTSUPP;
145 #endif
146
147 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev);
148 if (ret < 0)
149 goto err_rt_release;
150 dev_hold(route_dev);
151
152 if (!attr->ttl)
153 attr->ttl = ip4_dst_hoplimit(&rt->dst);
154 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr);
155 if (!n) {
156 ret = -ENOMEM;
157 goto err_dev_release;
158 }
159
160 ip_rt_put(rt);
161 attr->route_dev = route_dev;
162 attr->out_dev = out_dev;
163 attr->n = n;
164 return 0;
165
166 err_dev_release:
167 dev_put(route_dev);
168 err_rt_release:
169 ip_rt_put(rt);
170 return ret;
171 }
172
mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr * attr)173 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr)
174 {
175 mlx5e_tc_tun_route_attr_cleanup(attr);
176 }
177
mlx5e_netdev_kind(struct net_device * dev)178 static const char *mlx5e_netdev_kind(struct net_device *dev)
179 {
180 if (dev->rtnl_link_ops)
181 return dev->rtnl_link_ops->kind;
182 else
183 return "unknown";
184 }
185
mlx5e_gen_ip_tunnel_header(char buf[],__u8 * ip_proto,struct mlx5e_encap_entry * e)186 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
187 struct mlx5e_encap_entry *e)
188 {
189 if (!e->tunnel) {
190 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
191 return -EOPNOTSUPP;
192 }
193
194 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
195 }
196
gen_eth_tnl_hdr(char * buf,struct net_device * dev,struct mlx5e_encap_entry * e,u16 proto)197 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
198 struct mlx5e_encap_entry *e,
199 u16 proto)
200 {
201 struct ethhdr *eth = (struct ethhdr *)buf;
202 char *ip;
203
204 ether_addr_copy(eth->h_dest, e->h_dest);
205 ether_addr_copy(eth->h_source, dev->dev_addr);
206 if (is_vlan_dev(dev)) {
207 struct vlan_hdr *vlan = (struct vlan_hdr *)
208 ((char *)eth + ETH_HLEN);
209 ip = (char *)vlan + VLAN_HLEN;
210 eth->h_proto = vlan_dev_vlan_proto(dev);
211 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
212 vlan->h_vlan_encapsulated_proto = htons(proto);
213 } else {
214 eth->h_proto = htons(proto);
215 ip = (char *)eth + ETH_HLEN;
216 }
217
218 return ip;
219 }
220
mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)221 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
222 struct net_device *mirred_dev,
223 struct mlx5e_encap_entry *e)
224 {
225 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
226 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
227 struct mlx5_pkt_reformat_params reformat_params;
228 struct mlx5e_neigh m_neigh = {};
229 TC_TUN_ROUTE_ATTR_INIT(attr);
230 int ipv4_encap_size;
231 char *encap_header;
232 struct iphdr *ip;
233 u8 nud_state;
234 int err;
235
236 /* add the IP fields */
237 attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos);
238 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
239 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
240 attr.ttl = tun_key->ttl;
241
242 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
243 if (err)
244 return err;
245
246 ipv4_encap_size =
247 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
248 sizeof(struct iphdr) +
249 e->tunnel->calc_hlen(e);
250
251 if (max_encap_size < ipv4_encap_size) {
252 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
253 ipv4_encap_size, max_encap_size);
254 err = -EOPNOTSUPP;
255 goto release_neigh;
256 }
257
258 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
259 if (!encap_header) {
260 err = -ENOMEM;
261 goto release_neigh;
262 }
263
264 m_neigh.family = attr.n->ops->family;
265 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
266 e->out_dev = attr.out_dev;
267 e->route_dev_ifindex = attr.route_dev->ifindex;
268
269 /* It's important to add the neigh to the hash table before checking
270 * the neigh validity state. So if we'll get a notification, in case the
271 * neigh changes it's validity state, we would find the relevant neigh
272 * in the hash.
273 */
274 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
275 if (err)
276 goto free_encap;
277
278 read_lock_bh(&attr.n->lock);
279 nud_state = attr.n->nud_state;
280 ether_addr_copy(e->h_dest, attr.n->ha);
281 read_unlock_bh(&attr.n->lock);
282
283 /* add ethernet header */
284 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
285 ETH_P_IP);
286
287 /* add ip header */
288 ip->tos = tun_key->tos;
289 ip->version = 0x4;
290 ip->ihl = 0x5;
291 ip->ttl = attr.ttl;
292 ip->daddr = attr.fl.fl4.daddr;
293 ip->saddr = attr.fl.fl4.saddr;
294
295 /* add tunneling protocol header */
296 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
297 &ip->protocol, e);
298 if (err)
299 goto destroy_neigh_entry;
300
301 e->encap_size = ipv4_encap_size;
302 e->encap_header = encap_header;
303 encap_header = NULL;
304
305 if (!(nud_state & NUD_VALID)) {
306 neigh_event_send(attr.n, NULL);
307 /* the encap entry will be made valid on neigh update event
308 * and not used before that.
309 */
310 goto release_neigh;
311 }
312
313 memset(&reformat_params, 0, sizeof(reformat_params));
314 reformat_params.type = e->reformat_type;
315 reformat_params.size = e->encap_size;
316 reformat_params.data = e->encap_header;
317 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
318 MLX5_FLOW_NAMESPACE_FDB);
319 if (IS_ERR(e->pkt_reformat)) {
320 err = PTR_ERR(e->pkt_reformat);
321 goto destroy_neigh_entry;
322 }
323
324 e->flags |= MLX5_ENCAP_ENTRY_VALID;
325 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
326 mlx5e_route_lookup_ipv4_put(&attr);
327 return err;
328
329 destroy_neigh_entry:
330 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
331 free_encap:
332 kfree(encap_header);
333 release_neigh:
334 mlx5e_route_lookup_ipv4_put(&attr);
335 return err;
336 }
337
mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)338 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
339 struct net_device *mirred_dev,
340 struct mlx5e_encap_entry *e)
341 {
342 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
343 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
344 struct mlx5_pkt_reformat_params reformat_params;
345 TC_TUN_ROUTE_ATTR_INIT(attr);
346 int ipv4_encap_size;
347 char *encap_header;
348 struct iphdr *ip;
349 u8 nud_state;
350 int err;
351
352 /* add the IP fields */
353 attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos);
354 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
355 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
356 attr.ttl = tun_key->ttl;
357
358 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
359 if (err)
360 return err;
361
362 ipv4_encap_size =
363 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
364 sizeof(struct iphdr) +
365 e->tunnel->calc_hlen(e);
366
367 if (max_encap_size < ipv4_encap_size) {
368 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
369 ipv4_encap_size, max_encap_size);
370 err = -EOPNOTSUPP;
371 goto release_neigh;
372 }
373
374 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
375 if (!encap_header) {
376 err = -ENOMEM;
377 goto release_neigh;
378 }
379
380 e->route_dev_ifindex = attr.route_dev->ifindex;
381
382 read_lock_bh(&attr.n->lock);
383 nud_state = attr.n->nud_state;
384 ether_addr_copy(e->h_dest, attr.n->ha);
385 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
386 read_unlock_bh(&attr.n->lock);
387
388 /* add ethernet header */
389 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
390 ETH_P_IP);
391
392 /* add ip header */
393 ip->tos = tun_key->tos;
394 ip->version = 0x4;
395 ip->ihl = 0x5;
396 ip->ttl = attr.ttl;
397 ip->daddr = attr.fl.fl4.daddr;
398 ip->saddr = attr.fl.fl4.saddr;
399
400 /* add tunneling protocol header */
401 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
402 &ip->protocol, e);
403 if (err)
404 goto free_encap;
405
406 e->encap_size = ipv4_encap_size;
407 kfree(e->encap_header);
408 e->encap_header = encap_header;
409 encap_header = NULL;
410
411 if (!(nud_state & NUD_VALID)) {
412 neigh_event_send(attr.n, NULL);
413 /* the encap entry will be made valid on neigh update event
414 * and not used before that.
415 */
416 goto release_neigh;
417 }
418
419 memset(&reformat_params, 0, sizeof(reformat_params));
420 reformat_params.type = e->reformat_type;
421 reformat_params.size = e->encap_size;
422 reformat_params.data = e->encap_header;
423 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
424 MLX5_FLOW_NAMESPACE_FDB);
425 if (IS_ERR(e->pkt_reformat)) {
426 err = PTR_ERR(e->pkt_reformat);
427 goto free_encap;
428 }
429
430 e->flags |= MLX5_ENCAP_ENTRY_VALID;
431 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
432 mlx5e_route_lookup_ipv4_put(&attr);
433 return err;
434
435 free_encap:
436 kfree(encap_header);
437 release_neigh:
438 mlx5e_route_lookup_ipv4_put(&attr);
439 return err;
440 }
441
442 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
mlx5e_route_lookup_ipv6_get(struct mlx5e_priv * priv,struct net_device * dev,struct mlx5e_tc_tun_route_attr * attr)443 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv,
444 struct net_device *dev,
445 struct mlx5e_tc_tun_route_attr *attr)
446 {
447 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
448 struct net_device *route_dev;
449 struct net_device *out_dev;
450 struct dst_entry *dst;
451 struct neighbour *n;
452 int ret;
453
454 if (tunnel && tunnel->get_remote_ifindex)
455 attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(dev);
456 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6,
457 NULL);
458 if (IS_ERR(dst))
459 return PTR_ERR(dst);
460
461 if (!attr->ttl)
462 attr->ttl = ip6_dst_hoplimit(dst);
463
464 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev);
465 if (ret < 0)
466 goto err_dst_release;
467
468 dev_hold(route_dev);
469 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr);
470 if (!n) {
471 ret = -ENOMEM;
472 goto err_dev_release;
473 }
474
475 dst_release(dst);
476 attr->out_dev = out_dev;
477 attr->route_dev = route_dev;
478 attr->n = n;
479 return 0;
480
481 err_dev_release:
482 dev_put(route_dev);
483 err_dst_release:
484 dst_release(dst);
485 return ret;
486 }
487
mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr * attr)488 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr)
489 {
490 mlx5e_tc_tun_route_attr_cleanup(attr);
491 }
492
mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)493 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
494 struct net_device *mirred_dev,
495 struct mlx5e_encap_entry *e)
496 {
497 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
498 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
499 struct mlx5_pkt_reformat_params reformat_params;
500 struct mlx5e_neigh m_neigh = {};
501 TC_TUN_ROUTE_ATTR_INIT(attr);
502 struct ipv6hdr *ip6h;
503 int ipv6_encap_size;
504 char *encap_header;
505 u8 nud_state;
506 int err;
507
508 attr.ttl = tun_key->ttl;
509 attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
510 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
511 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
512
513 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
514 if (err)
515 return err;
516
517 ipv6_encap_size =
518 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
519 sizeof(struct ipv6hdr) +
520 e->tunnel->calc_hlen(e);
521
522 if (max_encap_size < ipv6_encap_size) {
523 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
524 ipv6_encap_size, max_encap_size);
525 err = -EOPNOTSUPP;
526 goto release_neigh;
527 }
528
529 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
530 if (!encap_header) {
531 err = -ENOMEM;
532 goto release_neigh;
533 }
534
535 m_neigh.family = attr.n->ops->family;
536 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
537 e->out_dev = attr.out_dev;
538 e->route_dev_ifindex = attr.route_dev->ifindex;
539
540 /* It's important to add the neigh to the hash table before checking
541 * the neigh validity state. So if we'll get a notification, in case the
542 * neigh changes it's validity state, we would find the relevant neigh
543 * in the hash.
544 */
545 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
546 if (err)
547 goto free_encap;
548
549 read_lock_bh(&attr.n->lock);
550 nud_state = attr.n->nud_state;
551 ether_addr_copy(e->h_dest, attr.n->ha);
552 read_unlock_bh(&attr.n->lock);
553
554 /* add ethernet header */
555 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
556 ETH_P_IPV6);
557
558 /* add ip header */
559 ip6_flow_hdr(ip6h, tun_key->tos, 0);
560 /* the HW fills up ipv6 payload len */
561 ip6h->hop_limit = attr.ttl;
562 ip6h->daddr = attr.fl.fl6.daddr;
563 ip6h->saddr = attr.fl.fl6.saddr;
564
565 /* add tunneling protocol header */
566 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
567 &ip6h->nexthdr, e);
568 if (err)
569 goto destroy_neigh_entry;
570
571 e->encap_size = ipv6_encap_size;
572 e->encap_header = encap_header;
573 encap_header = NULL;
574
575 if (!(nud_state & NUD_VALID)) {
576 neigh_event_send(attr.n, NULL);
577 /* the encap entry will be made valid on neigh update event
578 * and not used before that.
579 */
580 goto release_neigh;
581 }
582
583 memset(&reformat_params, 0, sizeof(reformat_params));
584 reformat_params.type = e->reformat_type;
585 reformat_params.size = e->encap_size;
586 reformat_params.data = e->encap_header;
587 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
588 MLX5_FLOW_NAMESPACE_FDB);
589 if (IS_ERR(e->pkt_reformat)) {
590 err = PTR_ERR(e->pkt_reformat);
591 goto destroy_neigh_entry;
592 }
593
594 e->flags |= MLX5_ENCAP_ENTRY_VALID;
595 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
596 mlx5e_route_lookup_ipv6_put(&attr);
597 return err;
598
599 destroy_neigh_entry:
600 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
601 free_encap:
602 kfree(encap_header);
603 release_neigh:
604 mlx5e_route_lookup_ipv6_put(&attr);
605 return err;
606 }
607
mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)608 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv,
609 struct net_device *mirred_dev,
610 struct mlx5e_encap_entry *e)
611 {
612 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
613 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
614 struct mlx5_pkt_reformat_params reformat_params;
615 TC_TUN_ROUTE_ATTR_INIT(attr);
616 struct ipv6hdr *ip6h;
617 int ipv6_encap_size;
618 char *encap_header;
619 u8 nud_state;
620 int err;
621
622 attr.ttl = tun_key->ttl;
623
624 attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
625 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
626 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
627
628 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
629 if (err)
630 return err;
631
632 ipv6_encap_size =
633 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
634 sizeof(struct ipv6hdr) +
635 e->tunnel->calc_hlen(e);
636
637 if (max_encap_size < ipv6_encap_size) {
638 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
639 ipv6_encap_size, max_encap_size);
640 err = -EOPNOTSUPP;
641 goto release_neigh;
642 }
643
644 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
645 if (!encap_header) {
646 err = -ENOMEM;
647 goto release_neigh;
648 }
649
650 e->route_dev_ifindex = attr.route_dev->ifindex;
651
652 read_lock_bh(&attr.n->lock);
653 nud_state = attr.n->nud_state;
654 ether_addr_copy(e->h_dest, attr.n->ha);
655 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
656 read_unlock_bh(&attr.n->lock);
657
658 /* add ethernet header */
659 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
660 ETH_P_IPV6);
661
662 /* add ip header */
663 ip6_flow_hdr(ip6h, tun_key->tos, 0);
664 /* the HW fills up ipv6 payload len */
665 ip6h->hop_limit = attr.ttl;
666 ip6h->daddr = attr.fl.fl6.daddr;
667 ip6h->saddr = attr.fl.fl6.saddr;
668
669 /* add tunneling protocol header */
670 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
671 &ip6h->nexthdr, e);
672 if (err)
673 goto free_encap;
674
675 e->encap_size = ipv6_encap_size;
676 kfree(e->encap_header);
677 e->encap_header = encap_header;
678 encap_header = NULL;
679
680 if (!(nud_state & NUD_VALID)) {
681 neigh_event_send(attr.n, NULL);
682 /* the encap entry will be made valid on neigh update event
683 * and not used before that.
684 */
685 goto release_neigh;
686 }
687
688 memset(&reformat_params, 0, sizeof(reformat_params));
689 reformat_params.type = e->reformat_type;
690 reformat_params.size = e->encap_size;
691 reformat_params.data = e->encap_header;
692 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
693 MLX5_FLOW_NAMESPACE_FDB);
694 if (IS_ERR(e->pkt_reformat)) {
695 err = PTR_ERR(e->pkt_reformat);
696 goto free_encap;
697 }
698
699 e->flags |= MLX5_ENCAP_ENTRY_VALID;
700 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
701 mlx5e_route_lookup_ipv6_put(&attr);
702 return err;
703
704 free_encap:
705 kfree(encap_header);
706 release_neigh:
707 mlx5e_route_lookup_ipv6_put(&attr);
708 return err;
709 }
710 #endif
711
mlx5e_tc_tun_route_lookup(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * flow_attr,struct net_device * filter_dev)712 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv,
713 struct mlx5_flow_spec *spec,
714 struct mlx5_flow_attr *flow_attr,
715 struct net_device *filter_dev)
716 {
717 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr;
718 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
719 struct mlx5e_tc_int_port *int_port;
720 TC_TUN_ROUTE_ATTR_INIT(attr);
721 u16 vport_num;
722 int err = 0;
723
724 if (flow_attr->tun_ip_version == 4) {
725 /* Addresses are swapped for decap */
726 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4;
727 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4;
728 err = mlx5e_route_lookup_ipv4_get(priv, filter_dev, &attr);
729 }
730 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
731 else if (flow_attr->tun_ip_version == 6) {
732 /* Addresses are swapped for decap */
733 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6;
734 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6;
735 err = mlx5e_route_lookup_ipv6_get(priv, filter_dev, &attr);
736 }
737 #endif
738 else
739 return 0;
740
741 if (err)
742 return err;
743
744 if (attr.route_dev->netdev_ops == &mlx5e_netdev_ops &&
745 mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) {
746 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num);
747 if (err)
748 goto out;
749
750 esw_attr->rx_tun_attr->decap_vport = vport_num;
751 } else if (netif_is_ovs_master(attr.route_dev) && mlx5e_tc_int_port_supported(esw)) {
752 int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv),
753 attr.route_dev->ifindex,
754 MLX5E_TC_INT_PORT_INGRESS);
755 if (IS_ERR(int_port)) {
756 err = PTR_ERR(int_port);
757 goto out;
758 }
759 esw_attr->int_port = int_port;
760 }
761
762 out:
763 if (flow_attr->tun_ip_version == 4)
764 mlx5e_route_lookup_ipv4_put(&attr);
765 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
766 else if (flow_attr->tun_ip_version == 6)
767 mlx5e_route_lookup_ipv6_put(&attr);
768 #endif
769 return err;
770 }
771
mlx5e_tc_tun_device_to_offload(struct mlx5e_priv * priv,struct net_device * netdev)772 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
773 struct net_device *netdev)
774 {
775 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);
776
777 if (tunnel && tunnel->can_offload(priv))
778 return true;
779 else
780 return false;
781 }
782
mlx5e_tc_tun_init_encap_attr(struct net_device * tunnel_dev,struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct netlink_ext_ack * extack)783 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
784 struct mlx5e_priv *priv,
785 struct mlx5e_encap_entry *e,
786 struct netlink_ext_ack *extack)
787 {
788 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev);
789
790 if (!tunnel) {
791 e->reformat_type = -1;
792 return -EOPNOTSUPP;
793 }
794
795 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack);
796 }
797
mlx5e_tc_tun_parse(struct net_device * filter_dev,struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,u8 * match_level)798 int mlx5e_tc_tun_parse(struct net_device *filter_dev,
799 struct mlx5e_priv *priv,
800 struct mlx5_flow_spec *spec,
801 struct flow_cls_offload *f,
802 u8 *match_level)
803 {
804 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev);
805 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
806 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
807 outer_headers);
808 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
809 outer_headers);
810 struct netlink_ext_ack *extack = f->common.extack;
811 int err = 0;
812
813 if (!tunnel) {
814 netdev_warn(priv->netdev,
815 "decapsulation offload is not supported for %s net device\n",
816 mlx5e_netdev_kind(filter_dev));
817 err = -EOPNOTSUPP;
818 goto out;
819 }
820
821 *match_level = tunnel->match_level;
822
823 if (tunnel->parse_udp_ports) {
824 err = tunnel->parse_udp_ports(priv, spec, f,
825 headers_c, headers_v);
826 if (err)
827 goto out;
828 }
829
830 if (tunnel->parse_tunnel) {
831 err = tunnel->parse_tunnel(priv, spec, f,
832 headers_c, headers_v);
833 if (err)
834 goto out;
835 }
836
837 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
838 struct flow_dissector_key_basic key_basic = {};
839 struct flow_dissector_key_basic mask_basic = {
840 .n_proto = htons(0xFFFF),
841 };
842 struct flow_match_basic match_basic = {
843 .key = &key_basic, .mask = &mask_basic,
844 };
845 struct flow_match_control match;
846 u16 addr_type;
847
848 flow_rule_match_enc_control(rule, &match);
849 addr_type = match.key->addr_type;
850
851 if (flow_rule_has_enc_control_flags(match.mask->flags,
852 extack)) {
853 err = -EOPNOTSUPP;
854 goto out;
855 }
856
857 /* For tunnel addr_type used same key id`s as for non-tunnel */
858 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
859 struct flow_match_ipv4_addrs match;
860
861 flow_rule_match_enc_ipv4_addrs(rule, &match);
862 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
863 src_ipv4_src_ipv6.ipv4_layout.ipv4,
864 ntohl(match.mask->src));
865 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
866 src_ipv4_src_ipv6.ipv4_layout.ipv4,
867 ntohl(match.key->src));
868
869 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
870 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
871 ntohl(match.mask->dst));
872 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
873 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
874 ntohl(match.key->dst));
875
876 key_basic.n_proto = htons(ETH_P_IP);
877 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
878 headers_c, headers_v);
879 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
880 struct flow_match_ipv6_addrs match;
881
882 flow_rule_match_enc_ipv6_addrs(rule, &match);
883 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
884 src_ipv4_src_ipv6.ipv6_layout.ipv6),
885 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
886 ipv6));
887 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
888 src_ipv4_src_ipv6.ipv6_layout.ipv6),
889 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
890 ipv6));
891
892 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
893 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
894 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
895 ipv6));
896 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
897 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
898 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
899 ipv6));
900
901 key_basic.n_proto = htons(ETH_P_IPV6);
902 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
903 headers_c, headers_v);
904 }
905 }
906
907 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
908 struct flow_match_ip match;
909
910 flow_rule_match_enc_ip(rule, &match);
911 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
912 match.mask->tos & 0x3);
913 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
914 match.key->tos & 0x3);
915
916 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
917 match.mask->tos >> 2);
918 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
919 match.key->tos >> 2);
920
921 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
922 match.mask->ttl);
923 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
924 match.key->ttl);
925
926 if (match.mask->ttl &&
927 !MLX5_CAP_ESW_FLOWTABLE_FDB
928 (priv->mdev,
929 ft_field_support.outer_ipv4_ttl)) {
930 NL_SET_ERR_MSG_MOD(extack,
931 "Matching on TTL is not supported");
932 err = -EOPNOTSUPP;
933 goto out;
934 }
935 }
936
937 /* let software handle IP fragments */
938 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
939 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
940
941 return 0;
942
943 out:
944 return err;
945 }
946
mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,void * headers_c,void * headers_v)947 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
948 struct mlx5_flow_spec *spec,
949 struct flow_cls_offload *f,
950 void *headers_c,
951 void *headers_v)
952 {
953 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
954 struct netlink_ext_ack *extack = f->common.extack;
955 struct flow_match_ports enc_ports;
956
957 /* Full udp dst port must be given */
958
959 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
960 NL_SET_ERR_MSG_MOD(extack,
961 "UDP tunnel decap filter must include enc_dst_port condition");
962 netdev_warn(priv->netdev,
963 "UDP tunnel decap filter must include enc_dst_port condition\n");
964 return -EOPNOTSUPP;
965 }
966
967 flow_rule_match_enc_ports(rule, &enc_ports);
968
969 if (memchr_inv(&enc_ports.mask->dst, 0xff,
970 sizeof(enc_ports.mask->dst))) {
971 NL_SET_ERR_MSG_MOD(extack,
972 "UDP tunnel decap filter must match enc_dst_port fully");
973 netdev_warn(priv->netdev,
974 "UDP tunnel decap filter must match enc_dst_port fully\n");
975 return -EOPNOTSUPP;
976 }
977
978 /* match on UDP protocol and dst port number */
979
980 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
981 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
982
983 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
984 ntohs(enc_ports.mask->dst));
985 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
986 ntohs(enc_ports.key->dst));
987
988 /* UDP src port on outer header is generated by HW,
989 * so it is probably a bad idea to request matching it.
990 * Nonetheless, it is allowed.
991 */
992
993 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
994 ntohs(enc_ports.mask->src));
995 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
996 ntohs(enc_ports.key->src));
997
998 return 0;
999 }
1000