1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * XFRM virtual interface 4 * 5 * Copyright (C) 2018 secunet Security Networks AG 6 * 7 * Author: 8 * Steffen Klassert <steffen.klassert@secunet.com> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/capability.h> 13 #include <linux/errno.h> 14 #include <linux/types.h> 15 #include <linux/sockios.h> 16 #include <linux/icmp.h> 17 #include <linux/if.h> 18 #include <linux/in.h> 19 #include <linux/ip.h> 20 #include <linux/net.h> 21 #include <linux/in6.h> 22 #include <linux/netdevice.h> 23 #include <linux/if_link.h> 24 #include <linux/if_arp.h> 25 #include <linux/icmpv6.h> 26 #include <linux/init.h> 27 #include <linux/route.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/netfilter_ipv6.h> 30 #include <linux/slab.h> 31 #include <linux/hash.h> 32 33 #include <linux/uaccess.h> 34 #include <linux/atomic.h> 35 36 #include <net/gso.h> 37 #include <net/icmp.h> 38 #include <net/ip.h> 39 #include <net/ipv6.h> 40 #include <net/ip6_route.h> 41 #include <net/ip_tunnels.h> 42 #include <net/addrconf.h> 43 #include <net/xfrm.h> 44 #include <net/net_namespace.h> 45 #include <net/dst_metadata.h> 46 #include <net/netns/generic.h> 47 #include <linux/etherdevice.h> 48 49 static int xfrmi_dev_init(struct net_device *dev); 50 static void xfrmi_dev_setup(struct net_device *dev); 51 static struct rtnl_link_ops xfrmi_link_ops __read_mostly; 52 static unsigned int xfrmi_net_id __read_mostly; 53 static const struct net_device_ops xfrmi_netdev_ops; 54 55 #define XFRMI_HASH_BITS 8 56 #define XFRMI_HASH_SIZE BIT(XFRMI_HASH_BITS) 57 58 struct xfrmi_net { 59 /* lists for storing interfaces in use */ 60 struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE]; 61 struct xfrm_if __rcu *collect_md_xfrmi; 62 }; 63 64 static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = { 65 [LWT_XFRM_IF_ID] = NLA_POLICY_MIN(NLA_U32, 1), 66 [LWT_XFRM_LINK] = NLA_POLICY_MIN(NLA_U32, 1), 67 }; 68 69 static void xfrmi_destroy_state(struct lwtunnel_state *lwt) 70 { 71 } 72 73 static int xfrmi_build_state(struct net *net, struct nlattr *nla, 74 unsigned int family, const void *cfg, 75 struct lwtunnel_state **ts, 76 struct netlink_ext_ack *extack) 77 { 78 struct nlattr *tb[LWT_XFRM_MAX + 1]; 79 struct lwtunnel_state *new_state; 80 struct xfrm_md_info *info; 81 int ret; 82 83 ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack); 84 if (ret < 0) 85 return ret; 86 87 if (!tb[LWT_XFRM_IF_ID]) { 88 NL_SET_ERR_MSG(extack, "if_id must be set"); 89 return -EINVAL; 90 } 91 92 new_state = lwtunnel_state_alloc(sizeof(*info)); 93 if (!new_state) { 94 NL_SET_ERR_MSG(extack, "failed to create encap info"); 95 return -ENOMEM; 96 } 97 98 new_state->type = LWTUNNEL_ENCAP_XFRM; 99 100 info = lwt_xfrm_info(new_state); 101 102 info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]); 103 104 if (tb[LWT_XFRM_LINK]) 105 info->link = nla_get_u32(tb[LWT_XFRM_LINK]); 106 107 *ts = new_state; 108 return 0; 109 } 110 111 static int xfrmi_fill_encap_info(struct sk_buff *skb, 112 struct lwtunnel_state *lwt) 113 { 114 struct xfrm_md_info *info = lwt_xfrm_info(lwt); 115 116 if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) || 117 (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link))) 118 return -EMSGSIZE; 119 120 return 0; 121 } 122 123 static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate) 124 { 125 return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */ 126 nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */ 127 } 128 129 static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b) 130 { 131 struct xfrm_md_info *a_info = lwt_xfrm_info(a); 132 struct xfrm_md_info *b_info = lwt_xfrm_info(b); 133 134 return memcmp(a_info, b_info, sizeof(*a_info)); 135 } 136 137 static const struct lwtunnel_encap_ops xfrmi_encap_ops = { 138 .build_state = xfrmi_build_state, 139 .destroy_state = xfrmi_destroy_state, 140 .fill_encap = xfrmi_fill_encap_info, 141 .get_encap_size = xfrmi_encap_nlsize, 142 .cmp_encap = xfrmi_encap_cmp, 143 .owner = THIS_MODULE, 144 }; 145 146 #define for_each_xfrmi_rcu(start, xi) \ 147 for (xi = rcu_dereference(start); xi; xi = rcu_dereference(xi->next)) 148 149 static u32 xfrmi_hash(u32 if_id) 150 { 151 return hash_32(if_id, XFRMI_HASH_BITS); 152 } 153 154 static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x) 155 { 156 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 157 struct xfrm_if *xi; 158 159 for_each_xfrmi_rcu(xfrmn->xfrmi[xfrmi_hash(x->if_id)], xi) { 160 if (x->if_id == xi->p.if_id && 161 (xi->dev->flags & IFF_UP)) 162 return xi; 163 } 164 165 xi = rcu_dereference(xfrmn->collect_md_xfrmi); 166 if (xi && (xi->dev->flags & IFF_UP)) 167 return xi; 168 169 return NULL; 170 } 171 172 static bool xfrmi_decode_session(struct sk_buff *skb, 173 unsigned short family, 174 struct xfrm_if_decode_session_result *res) 175 { 176 struct net_device *dev; 177 struct xfrm_if *xi; 178 int ifindex = 0; 179 180 if (!secpath_exists(skb) || !skb->dev) 181 return false; 182 183 switch (family) { 184 case AF_INET6: 185 ifindex = inet6_sdif(skb); 186 break; 187 case AF_INET: 188 ifindex = inet_sdif(skb); 189 break; 190 } 191 192 if (ifindex) { 193 struct net *net = xs_net(xfrm_input_state(skb)); 194 195 dev = dev_get_by_index_rcu(net, ifindex); 196 } else { 197 dev = skb->dev; 198 } 199 200 if (!dev || !(dev->flags & IFF_UP)) 201 return false; 202 if (dev->netdev_ops != &xfrmi_netdev_ops) 203 return false; 204 205 xi = netdev_priv(dev); 206 res->net = xi->net; 207 208 if (xi->p.collect_md) 209 res->if_id = xfrm_input_state(skb)->if_id; 210 else 211 res->if_id = xi->p.if_id; 212 return true; 213 } 214 215 static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi) 216 { 217 struct xfrm_if __rcu **xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)]; 218 219 rcu_assign_pointer(xi->next , rtnl_dereference(*xip)); 220 rcu_assign_pointer(*xip, xi); 221 } 222 223 static void xfrmi_unlink(struct xfrmi_net *xfrmn, struct xfrm_if *xi) 224 { 225 struct xfrm_if __rcu **xip; 226 struct xfrm_if *iter; 227 228 for (xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)]; 229 (iter = rtnl_dereference(*xip)) != NULL; 230 xip = &iter->next) { 231 if (xi == iter) { 232 rcu_assign_pointer(*xip, xi->next); 233 break; 234 } 235 } 236 } 237 238 static void xfrmi_dev_free(struct net_device *dev) 239 { 240 struct xfrm_if *xi = netdev_priv(dev); 241 242 gro_cells_destroy(&xi->gro_cells); 243 } 244 245 static int xfrmi_create(struct net *net, struct net_device *dev) 246 { 247 struct xfrm_if *xi = netdev_priv(dev); 248 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 249 int err; 250 251 dev->rtnl_link_ops = &xfrmi_link_ops; 252 err = register_netdevice(dev); 253 if (err < 0) 254 goto out; 255 256 if (xi->p.collect_md) 257 rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi); 258 else 259 xfrmi_link(xfrmn, xi); 260 261 return 0; 262 263 out: 264 return err; 265 } 266 267 static struct xfrm_if *xfrmi_locate(struct net *net, struct xfrm_if_parms *p) 268 { 269 struct xfrm_if __rcu **xip; 270 struct xfrm_if *xi; 271 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 272 273 for (xip = &xfrmn->xfrmi[xfrmi_hash(p->if_id)]; 274 (xi = rtnl_dereference(*xip)) != NULL; 275 xip = &xi->next) 276 if (xi->p.if_id == p->if_id) 277 return xi; 278 279 return NULL; 280 } 281 282 static void xfrmi_dev_uninit(struct net_device *dev) 283 { 284 struct xfrm_if *xi = netdev_priv(dev); 285 struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id); 286 287 if (xi->p.collect_md) 288 RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL); 289 else 290 xfrmi_unlink(xfrmn, xi); 291 } 292 293 static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet) 294 { 295 skb_clear_tstamp(skb); 296 skb->pkt_type = PACKET_HOST; 297 skb->skb_iif = 0; 298 skb->ignore_df = 0; 299 skb_dst_drop(skb); 300 nf_reset_ct(skb); 301 nf_reset_trace(skb); 302 303 if (!xnet) 304 return; 305 306 ipvs_reset(skb); 307 secpath_reset(skb); 308 skb_orphan(skb); 309 skb->mark = 0; 310 } 311 312 static int xfrmi_input(struct sk_buff *skb, int nexthdr, __be32 spi, 313 int encap_type, unsigned short family) 314 { 315 struct sec_path *sp; 316 317 sp = skb_sec_path(skb); 318 if (sp && (sp->len || sp->olen) && 319 !xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, family)) 320 goto discard; 321 322 XFRM_SPI_SKB_CB(skb)->family = family; 323 if (family == AF_INET) { 324 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr); 325 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL; 326 } else { 327 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr); 328 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL; 329 } 330 331 return xfrm_input(skb, nexthdr, spi, encap_type); 332 discard: 333 kfree_skb(skb); 334 return 0; 335 } 336 337 static int xfrmi4_rcv(struct sk_buff *skb) 338 { 339 return xfrmi_input(skb, ip_hdr(skb)->protocol, 0, 0, AF_INET); 340 } 341 342 static int xfrmi6_rcv(struct sk_buff *skb) 343 { 344 return xfrmi_input(skb, skb_network_header(skb)[IP6CB(skb)->nhoff], 345 0, 0, AF_INET6); 346 } 347 348 static int xfrmi4_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 349 { 350 return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET); 351 } 352 353 static int xfrmi6_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 354 { 355 return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET6); 356 } 357 358 static int xfrmi_rcv_cb(struct sk_buff *skb, int err) 359 { 360 const struct xfrm_mode *inner_mode; 361 struct net_device *dev; 362 struct xfrm_state *x; 363 struct xfrm_if *xi; 364 bool xnet; 365 int link; 366 367 if (err && !secpath_exists(skb)) 368 return 0; 369 370 x = xfrm_input_state(skb); 371 372 xi = xfrmi_lookup(xs_net(x), x); 373 if (!xi) 374 return 1; 375 376 link = skb->dev->ifindex; 377 dev = xi->dev; 378 skb->dev = dev; 379 380 if (err) { 381 DEV_STATS_INC(dev, rx_errors); 382 DEV_STATS_INC(dev, rx_dropped); 383 384 return 0; 385 } 386 387 xnet = !net_eq(xi->net, dev_net(skb->dev)); 388 389 if (xnet) { 390 inner_mode = &x->inner_mode; 391 392 if (x->sel.family == AF_UNSPEC) { 393 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); 394 if (inner_mode == NULL) { 395 XFRM_INC_STATS(dev_net(skb->dev), 396 LINUX_MIB_XFRMINSTATEMODEERROR); 397 return -EINVAL; 398 } 399 } 400 401 if (!xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, 402 inner_mode->family)) 403 return -EPERM; 404 } 405 406 xfrmi_scrub_packet(skb, xnet); 407 if (xi->p.collect_md) { 408 struct metadata_dst *md_dst; 409 410 md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC); 411 if (!md_dst) 412 return -ENOMEM; 413 414 md_dst->u.xfrm_info.if_id = x->if_id; 415 md_dst->u.xfrm_info.link = link; 416 skb_dst_set(skb, (struct dst_entry *)md_dst); 417 } 418 dev_sw_netstats_rx_add(dev, skb->len); 419 420 return 0; 421 } 422 423 static int 424 xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) 425 { 426 struct xfrm_if *xi = netdev_priv(dev); 427 struct dst_entry *dst = skb_dst(skb); 428 unsigned int length = skb->len; 429 struct net_device *tdev; 430 struct xfrm_state *x; 431 int err = -1; 432 u32 if_id; 433 int mtu; 434 435 if (xi->p.collect_md) { 436 struct xfrm_md_info *md_info = skb_xfrm_md_info(skb); 437 438 if (unlikely(!md_info)) 439 return -EINVAL; 440 441 if_id = md_info->if_id; 442 fl->flowi_oif = md_info->link; 443 if (md_info->dst_orig) { 444 struct dst_entry *tmp_dst = dst; 445 446 dst = md_info->dst_orig; 447 skb_dst_set(skb, dst); 448 md_info->dst_orig = NULL; 449 dst_release(tmp_dst); 450 } 451 } else { 452 if_id = xi->p.if_id; 453 } 454 455 dst_hold(dst); 456 dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id); 457 if (IS_ERR(dst)) { 458 err = PTR_ERR(dst); 459 dst = NULL; 460 goto tx_err_link_failure; 461 } 462 463 x = dst->xfrm; 464 if (!x) 465 goto tx_err_link_failure; 466 467 if (x->if_id != if_id) 468 goto tx_err_link_failure; 469 470 tdev = dst->dev; 471 472 if (tdev == dev) { 473 DEV_STATS_INC(dev, collisions); 474 net_warn_ratelimited("%s: Local routing loop detected!\n", 475 dev->name); 476 goto tx_err_dst_release; 477 } 478 479 mtu = dst_mtu(dst); 480 if ((!skb_is_gso(skb) && skb->len > mtu) || 481 (skb_is_gso(skb) && !skb_gso_validate_network_len(skb, mtu))) { 482 skb_dst_update_pmtu_no_confirm(skb, mtu); 483 484 if (skb->protocol == htons(ETH_P_IPV6)) { 485 if (mtu < IPV6_MIN_MTU) 486 mtu = IPV6_MIN_MTU; 487 488 if (skb->len > 1280) 489 icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 490 else 491 goto xmit; 492 } else { 493 if (!(ip_hdr(skb)->frag_off & htons(IP_DF))) 494 goto xmit; 495 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 496 htonl(mtu)); 497 } 498 499 dst_release(dst); 500 return -EMSGSIZE; 501 } 502 503 xmit: 504 xfrmi_scrub_packet(skb, !net_eq(xi->net, dev_net(dev))); 505 skb_dst_set(skb, dst); 506 skb->dev = tdev; 507 508 err = dst_output(xi->net, skb_to_full_sk(skb), skb); 509 if (net_xmit_eval(err) == 0) { 510 dev_sw_netstats_tx_add(dev, 1, length); 511 } else { 512 DEV_STATS_INC(dev, tx_errors); 513 DEV_STATS_INC(dev, tx_aborted_errors); 514 } 515 516 return 0; 517 tx_err_link_failure: 518 DEV_STATS_INC(dev, tx_carrier_errors); 519 dst_link_failure(skb); 520 tx_err_dst_release: 521 dst_release(dst); 522 return err; 523 } 524 525 static netdev_tx_t xfrmi_xmit(struct sk_buff *skb, struct net_device *dev) 526 { 527 struct xfrm_if *xi = netdev_priv(dev); 528 struct dst_entry *dst = skb_dst(skb); 529 struct flowi fl; 530 int ret; 531 532 memset(&fl, 0, sizeof(fl)); 533 534 switch (skb->protocol) { 535 case htons(ETH_P_IPV6): 536 memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); 537 xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET6); 538 if (!dst) { 539 fl.u.ip6.flowi6_oif = dev->ifindex; 540 fl.u.ip6.flowi6_flags |= FLOWI_FLAG_ANYSRC; 541 dst = ip6_route_output(dev_net(dev), NULL, &fl.u.ip6); 542 if (dst->error) { 543 dst_release(dst); 544 DEV_STATS_INC(dev, tx_carrier_errors); 545 goto tx_err; 546 } 547 skb_dst_set(skb, dst); 548 } 549 break; 550 case htons(ETH_P_IP): 551 memset(IPCB(skb), 0, sizeof(*IPCB(skb))); 552 xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET); 553 if (!dst) { 554 struct rtable *rt; 555 556 fl.u.ip4.flowi4_oif = dev->ifindex; 557 fl.u.ip4.flowi4_flags |= FLOWI_FLAG_ANYSRC; 558 rt = __ip_route_output_key(dev_net(dev), &fl.u.ip4); 559 if (IS_ERR(rt)) { 560 DEV_STATS_INC(dev, tx_carrier_errors); 561 goto tx_err; 562 } 563 skb_dst_set(skb, &rt->dst); 564 } 565 break; 566 default: 567 goto tx_err; 568 } 569 570 fl.flowi_oif = xi->p.link; 571 572 ret = xfrmi_xmit2(skb, dev, &fl); 573 if (ret < 0) 574 goto tx_err; 575 576 return NETDEV_TX_OK; 577 578 tx_err: 579 DEV_STATS_INC(dev, tx_errors); 580 DEV_STATS_INC(dev, tx_dropped); 581 kfree_skb(skb); 582 return NETDEV_TX_OK; 583 } 584 585 static int xfrmi4_err(struct sk_buff *skb, u32 info) 586 { 587 const struct iphdr *iph = (const struct iphdr *)skb->data; 588 struct net *net = dev_net(skb->dev); 589 int protocol = iph->protocol; 590 struct ip_comp_hdr *ipch; 591 struct ip_esp_hdr *esph; 592 struct ip_auth_hdr *ah ; 593 struct xfrm_state *x; 594 struct xfrm_if *xi; 595 __be32 spi; 596 597 switch (protocol) { 598 case IPPROTO_ESP: 599 esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2)); 600 spi = esph->spi; 601 break; 602 case IPPROTO_AH: 603 ah = (struct ip_auth_hdr *)(skb->data+(iph->ihl<<2)); 604 spi = ah->spi; 605 break; 606 case IPPROTO_COMP: 607 ipch = (struct ip_comp_hdr *)(skb->data+(iph->ihl<<2)); 608 spi = htonl(ntohs(ipch->cpi)); 609 break; 610 default: 611 return 0; 612 } 613 614 switch (icmp_hdr(skb)->type) { 615 case ICMP_DEST_UNREACH: 616 if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) 617 return 0; 618 break; 619 case ICMP_REDIRECT: 620 break; 621 default: 622 return 0; 623 } 624 625 x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 626 spi, protocol, AF_INET); 627 if (!x) 628 return 0; 629 630 xi = xfrmi_lookup(net, x); 631 if (!xi) { 632 xfrm_state_put(x); 633 return -1; 634 } 635 636 if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH) 637 ipv4_update_pmtu(skb, net, info, 0, protocol); 638 else 639 ipv4_redirect(skb, net, 0, protocol); 640 xfrm_state_put(x); 641 642 return 0; 643 } 644 645 static int xfrmi6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 646 u8 type, u8 code, int offset, __be32 info) 647 { 648 const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data; 649 struct net *net = dev_net(skb->dev); 650 int protocol = iph->nexthdr; 651 struct ip_comp_hdr *ipch; 652 struct ip_esp_hdr *esph; 653 struct ip_auth_hdr *ah; 654 struct xfrm_state *x; 655 struct xfrm_if *xi; 656 __be32 spi; 657 658 switch (protocol) { 659 case IPPROTO_ESP: 660 esph = (struct ip_esp_hdr *)(skb->data + offset); 661 spi = esph->spi; 662 break; 663 case IPPROTO_AH: 664 ah = (struct ip_auth_hdr *)(skb->data + offset); 665 spi = ah->spi; 666 break; 667 case IPPROTO_COMP: 668 ipch = (struct ip_comp_hdr *)(skb->data + offset); 669 spi = htonl(ntohs(ipch->cpi)); 670 break; 671 default: 672 return 0; 673 } 674 675 if (type != ICMPV6_PKT_TOOBIG && 676 type != NDISC_REDIRECT) 677 return 0; 678 679 x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 680 spi, protocol, AF_INET6); 681 if (!x) 682 return 0; 683 684 xi = xfrmi_lookup(net, x); 685 if (!xi) { 686 xfrm_state_put(x); 687 return -1; 688 } 689 690 if (type == NDISC_REDIRECT) 691 ip6_redirect(skb, net, skb->dev->ifindex, 0, 692 sock_net_uid(net, NULL)); 693 else 694 ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); 695 xfrm_state_put(x); 696 697 return 0; 698 } 699 700 static int xfrmi_change(struct xfrm_if *xi, const struct xfrm_if_parms *p) 701 { 702 if (xi->p.link != p->link) 703 return -EINVAL; 704 705 xi->p.if_id = p->if_id; 706 707 return 0; 708 } 709 710 static int xfrmi_update(struct xfrm_if *xi, struct xfrm_if_parms *p) 711 { 712 struct net *net = xi->net; 713 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 714 int err; 715 716 xfrmi_unlink(xfrmn, xi); 717 synchronize_net(); 718 err = xfrmi_change(xi, p); 719 xfrmi_link(xfrmn, xi); 720 netdev_state_change(xi->dev); 721 return err; 722 } 723 724 static int xfrmi_get_iflink(const struct net_device *dev) 725 { 726 struct xfrm_if *xi = netdev_priv(dev); 727 728 return READ_ONCE(xi->p.link); 729 } 730 731 static const struct net_device_ops xfrmi_netdev_ops = { 732 .ndo_init = xfrmi_dev_init, 733 .ndo_uninit = xfrmi_dev_uninit, 734 .ndo_start_xmit = xfrmi_xmit, 735 .ndo_get_stats64 = dev_get_tstats64, 736 .ndo_get_iflink = xfrmi_get_iflink, 737 }; 738 739 static void xfrmi_dev_setup(struct net_device *dev) 740 { 741 dev->netdev_ops = &xfrmi_netdev_ops; 742 dev->header_ops = &ip_tunnel_header_ops; 743 dev->type = ARPHRD_NONE; 744 dev->mtu = ETH_DATA_LEN; 745 dev->min_mtu = ETH_MIN_MTU; 746 dev->max_mtu = IP_MAX_MTU; 747 dev->flags = IFF_NOARP; 748 dev->needs_free_netdev = true; 749 dev->priv_destructor = xfrmi_dev_free; 750 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 751 netif_keep_dst(dev); 752 753 eth_broadcast_addr(dev->broadcast); 754 } 755 756 #define XFRMI_FEATURES (NETIF_F_SG | \ 757 NETIF_F_FRAGLIST | \ 758 NETIF_F_GSO_SOFTWARE | \ 759 NETIF_F_HW_CSUM) 760 761 static int xfrmi_dev_init(struct net_device *dev) 762 { 763 struct xfrm_if *xi = netdev_priv(dev); 764 struct net_device *phydev = __dev_get_by_index(xi->net, xi->p.link); 765 int err; 766 767 err = gro_cells_init(&xi->gro_cells, dev); 768 if (err) 769 return err; 770 771 dev->lltx = true; 772 dev->features |= XFRMI_FEATURES; 773 dev->hw_features |= XFRMI_FEATURES; 774 775 if (phydev) { 776 dev->needed_headroom = phydev->needed_headroom; 777 dev->needed_tailroom = phydev->needed_tailroom; 778 779 if (is_zero_ether_addr(dev->dev_addr)) 780 eth_hw_addr_inherit(dev, phydev); 781 if (is_zero_ether_addr(dev->broadcast)) 782 memcpy(dev->broadcast, phydev->broadcast, 783 dev->addr_len); 784 } else { 785 eth_hw_addr_random(dev); 786 eth_broadcast_addr(dev->broadcast); 787 } 788 789 return 0; 790 } 791 792 static int xfrmi_validate(struct nlattr *tb[], struct nlattr *data[], 793 struct netlink_ext_ack *extack) 794 { 795 return 0; 796 } 797 798 static void xfrmi_netlink_parms(struct nlattr *data[], 799 struct xfrm_if_parms *parms) 800 { 801 memset(parms, 0, sizeof(*parms)); 802 803 if (!data) 804 return; 805 806 if (data[IFLA_XFRM_LINK]) 807 parms->link = nla_get_u32(data[IFLA_XFRM_LINK]); 808 809 if (data[IFLA_XFRM_IF_ID]) 810 parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]); 811 812 if (data[IFLA_XFRM_COLLECT_METADATA]) 813 parms->collect_md = true; 814 } 815 816 static int xfrmi_newlink(struct net_device *dev, 817 struct rtnl_newlink_params *params, 818 struct netlink_ext_ack *extack) 819 { 820 struct nlattr **data = params->data; 821 struct xfrm_if_parms p = {}; 822 struct xfrm_if *xi; 823 struct net *net; 824 int err; 825 826 net = params->link_net ? : dev_net(dev); 827 xfrmi_netlink_parms(data, &p); 828 if (p.collect_md) { 829 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 830 831 if (p.link || p.if_id) { 832 NL_SET_ERR_MSG(extack, "link and if_id must be zero"); 833 return -EINVAL; 834 } 835 836 if (rtnl_dereference(xfrmn->collect_md_xfrmi)) 837 return -EEXIST; 838 839 } else { 840 if (!p.if_id) { 841 NL_SET_ERR_MSG(extack, "if_id must be non zero"); 842 return -EINVAL; 843 } 844 845 xi = xfrmi_locate(net, &p); 846 if (xi) 847 return -EEXIST; 848 } 849 850 xi = netdev_priv(dev); 851 xi->p = p; 852 xi->net = net; 853 xi->dev = dev; 854 855 err = xfrmi_create(net, dev); 856 return err; 857 } 858 859 static void xfrmi_dellink(struct net_device *dev, struct list_head *head) 860 { 861 unregister_netdevice_queue(dev, head); 862 } 863 864 static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], 865 struct nlattr *data[], 866 struct netlink_ext_ack *extack) 867 { 868 struct xfrm_if *xi = netdev_priv(dev); 869 struct net *net = xi->net; 870 struct xfrm_if_parms p = {}; 871 872 xfrmi_netlink_parms(data, &p); 873 if (!p.if_id) { 874 NL_SET_ERR_MSG(extack, "if_id must be non zero"); 875 return -EINVAL; 876 } 877 878 if (p.collect_md) { 879 NL_SET_ERR_MSG(extack, "collect_md can't be changed"); 880 return -EINVAL; 881 } 882 883 xi = xfrmi_locate(net, &p); 884 if (!xi) { 885 xi = netdev_priv(dev); 886 } else { 887 if (xi->dev != dev) 888 return -EEXIST; 889 if (xi->p.collect_md) { 890 NL_SET_ERR_MSG(extack, 891 "device can't be changed to collect_md"); 892 return -EINVAL; 893 } 894 } 895 896 return xfrmi_update(xi, &p); 897 } 898 899 static size_t xfrmi_get_size(const struct net_device *dev) 900 { 901 return 902 /* IFLA_XFRM_LINK */ 903 nla_total_size(4) + 904 /* IFLA_XFRM_IF_ID */ 905 nla_total_size(4) + 906 /* IFLA_XFRM_COLLECT_METADATA */ 907 nla_total_size(0) + 908 0; 909 } 910 911 static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev) 912 { 913 struct xfrm_if *xi = netdev_priv(dev); 914 struct xfrm_if_parms *parm = &xi->p; 915 916 if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) || 917 nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) || 918 (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA))) 919 goto nla_put_failure; 920 return 0; 921 922 nla_put_failure: 923 return -EMSGSIZE; 924 } 925 926 static struct net *xfrmi_get_link_net(const struct net_device *dev) 927 { 928 struct xfrm_if *xi = netdev_priv(dev); 929 930 return READ_ONCE(xi->net); 931 } 932 933 static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = { 934 [IFLA_XFRM_UNSPEC] = { .strict_start_type = IFLA_XFRM_COLLECT_METADATA }, 935 [IFLA_XFRM_LINK] = { .type = NLA_U32 }, 936 [IFLA_XFRM_IF_ID] = { .type = NLA_U32 }, 937 [IFLA_XFRM_COLLECT_METADATA] = { .type = NLA_FLAG }, 938 }; 939 940 static struct rtnl_link_ops xfrmi_link_ops __read_mostly = { 941 .kind = "xfrm", 942 .maxtype = IFLA_XFRM_MAX, 943 .policy = xfrmi_policy, 944 .priv_size = sizeof(struct xfrm_if), 945 .setup = xfrmi_dev_setup, 946 .validate = xfrmi_validate, 947 .newlink = xfrmi_newlink, 948 .dellink = xfrmi_dellink, 949 .changelink = xfrmi_changelink, 950 .get_size = xfrmi_get_size, 951 .fill_info = xfrmi_fill_info, 952 .get_link_net = xfrmi_get_link_net, 953 }; 954 955 static void __net_exit xfrmi_exit_rtnl(struct net *net, 956 struct list_head *dev_to_kill) 957 { 958 struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); 959 struct xfrm_if __rcu **xip; 960 struct xfrm_if *xi; 961 int i; 962 963 for (i = 0; i < XFRMI_HASH_SIZE; i++) { 964 for (xip = &xfrmn->xfrmi[i]; 965 (xi = rtnl_net_dereference(net, *xip)) != NULL; 966 xip = &xi->next) 967 unregister_netdevice_queue(xi->dev, dev_to_kill); 968 } 969 970 xi = rtnl_net_dereference(net, xfrmn->collect_md_xfrmi); 971 if (xi) 972 unregister_netdevice_queue(xi->dev, dev_to_kill); 973 } 974 975 static struct pernet_operations xfrmi_net_ops = { 976 .exit_rtnl = xfrmi_exit_rtnl, 977 .id = &xfrmi_net_id, 978 .size = sizeof(struct xfrmi_net), 979 }; 980 981 static struct xfrm6_protocol xfrmi_esp6_protocol __read_mostly = { 982 .handler = xfrmi6_rcv, 983 .input_handler = xfrmi6_input, 984 .cb_handler = xfrmi_rcv_cb, 985 .err_handler = xfrmi6_err, 986 .priority = 10, 987 }; 988 989 static struct xfrm6_protocol xfrmi_ah6_protocol __read_mostly = { 990 .handler = xfrm6_rcv, 991 .input_handler = xfrm_input, 992 .cb_handler = xfrmi_rcv_cb, 993 .err_handler = xfrmi6_err, 994 .priority = 10, 995 }; 996 997 static struct xfrm6_protocol xfrmi_ipcomp6_protocol __read_mostly = { 998 .handler = xfrm6_rcv, 999 .input_handler = xfrm_input, 1000 .cb_handler = xfrmi_rcv_cb, 1001 .err_handler = xfrmi6_err, 1002 .priority = 10, 1003 }; 1004 1005 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1006 static int xfrmi6_rcv_tunnel(struct sk_buff *skb) 1007 { 1008 const xfrm_address_t *saddr; 1009 __be32 spi; 1010 1011 saddr = (const xfrm_address_t *)&ipv6_hdr(skb)->saddr; 1012 spi = xfrm6_tunnel_spi_lookup(dev_net(skb->dev), saddr); 1013 1014 return xfrm6_rcv_spi(skb, IPPROTO_IPV6, spi, NULL); 1015 } 1016 1017 static struct xfrm6_tunnel xfrmi_ipv6_handler __read_mostly = { 1018 .handler = xfrmi6_rcv_tunnel, 1019 .cb_handler = xfrmi_rcv_cb, 1020 .err_handler = xfrmi6_err, 1021 .priority = 2, 1022 }; 1023 1024 static struct xfrm6_tunnel xfrmi_ip6ip_handler __read_mostly = { 1025 .handler = xfrmi6_rcv_tunnel, 1026 .cb_handler = xfrmi_rcv_cb, 1027 .err_handler = xfrmi6_err, 1028 .priority = 2, 1029 }; 1030 #endif 1031 1032 static struct xfrm4_protocol xfrmi_esp4_protocol __read_mostly = { 1033 .handler = xfrmi4_rcv, 1034 .input_handler = xfrmi4_input, 1035 .cb_handler = xfrmi_rcv_cb, 1036 .err_handler = xfrmi4_err, 1037 .priority = 10, 1038 }; 1039 1040 static struct xfrm4_protocol xfrmi_ah4_protocol __read_mostly = { 1041 .handler = xfrm4_rcv, 1042 .input_handler = xfrm_input, 1043 .cb_handler = xfrmi_rcv_cb, 1044 .err_handler = xfrmi4_err, 1045 .priority = 10, 1046 }; 1047 1048 static struct xfrm4_protocol xfrmi_ipcomp4_protocol __read_mostly = { 1049 .handler = xfrm4_rcv, 1050 .input_handler = xfrm_input, 1051 .cb_handler = xfrmi_rcv_cb, 1052 .err_handler = xfrmi4_err, 1053 .priority = 10, 1054 }; 1055 1056 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1057 static int xfrmi4_rcv_tunnel(struct sk_buff *skb) 1058 { 1059 return xfrm4_rcv_spi(skb, IPPROTO_IPIP, ip_hdr(skb)->saddr); 1060 } 1061 1062 static struct xfrm_tunnel xfrmi_ipip_handler __read_mostly = { 1063 .handler = xfrmi4_rcv_tunnel, 1064 .cb_handler = xfrmi_rcv_cb, 1065 .err_handler = xfrmi4_err, 1066 .priority = 3, 1067 }; 1068 1069 static struct xfrm_tunnel xfrmi_ipip6_handler __read_mostly = { 1070 .handler = xfrmi4_rcv_tunnel, 1071 .cb_handler = xfrmi_rcv_cb, 1072 .err_handler = xfrmi4_err, 1073 .priority = 2, 1074 }; 1075 #endif 1076 1077 static int __init xfrmi4_init(void) 1078 { 1079 int err; 1080 1081 err = xfrm4_protocol_register(&xfrmi_esp4_protocol, IPPROTO_ESP); 1082 if (err < 0) 1083 goto xfrm_proto_esp_failed; 1084 err = xfrm4_protocol_register(&xfrmi_ah4_protocol, IPPROTO_AH); 1085 if (err < 0) 1086 goto xfrm_proto_ah_failed; 1087 err = xfrm4_protocol_register(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1088 if (err < 0) 1089 goto xfrm_proto_comp_failed; 1090 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1091 err = xfrm4_tunnel_register(&xfrmi_ipip_handler, AF_INET); 1092 if (err < 0) 1093 goto xfrm_tunnel_ipip_failed; 1094 err = xfrm4_tunnel_register(&xfrmi_ipip6_handler, AF_INET6); 1095 if (err < 0) 1096 goto xfrm_tunnel_ipip6_failed; 1097 #endif 1098 1099 return 0; 1100 1101 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1102 xfrm_tunnel_ipip6_failed: 1103 xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET); 1104 xfrm_tunnel_ipip_failed: 1105 xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1106 #endif 1107 xfrm_proto_comp_failed: 1108 xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH); 1109 xfrm_proto_ah_failed: 1110 xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP); 1111 xfrm_proto_esp_failed: 1112 return err; 1113 } 1114 1115 static void xfrmi4_fini(void) 1116 { 1117 #if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL) 1118 xfrm4_tunnel_deregister(&xfrmi_ipip6_handler, AF_INET6); 1119 xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET); 1120 #endif 1121 xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP); 1122 xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH); 1123 xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP); 1124 } 1125 1126 static int __init xfrmi6_init(void) 1127 { 1128 int err; 1129 1130 err = xfrm6_protocol_register(&xfrmi_esp6_protocol, IPPROTO_ESP); 1131 if (err < 0) 1132 goto xfrm_proto_esp_failed; 1133 err = xfrm6_protocol_register(&xfrmi_ah6_protocol, IPPROTO_AH); 1134 if (err < 0) 1135 goto xfrm_proto_ah_failed; 1136 err = xfrm6_protocol_register(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1137 if (err < 0) 1138 goto xfrm_proto_comp_failed; 1139 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1140 err = xfrm6_tunnel_register(&xfrmi_ipv6_handler, AF_INET6); 1141 if (err < 0) 1142 goto xfrm_tunnel_ipv6_failed; 1143 err = xfrm6_tunnel_register(&xfrmi_ip6ip_handler, AF_INET); 1144 if (err < 0) 1145 goto xfrm_tunnel_ip6ip_failed; 1146 #endif 1147 1148 return 0; 1149 1150 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1151 xfrm_tunnel_ip6ip_failed: 1152 xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6); 1153 xfrm_tunnel_ipv6_failed: 1154 xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1155 #endif 1156 xfrm_proto_comp_failed: 1157 xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH); 1158 xfrm_proto_ah_failed: 1159 xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP); 1160 xfrm_proto_esp_failed: 1161 return err; 1162 } 1163 1164 static void xfrmi6_fini(void) 1165 { 1166 #if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL) 1167 xfrm6_tunnel_deregister(&xfrmi_ip6ip_handler, AF_INET); 1168 xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6); 1169 #endif 1170 xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP); 1171 xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH); 1172 xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP); 1173 } 1174 1175 static const struct xfrm_if_cb xfrm_if_cb = { 1176 .decode_session = xfrmi_decode_session, 1177 }; 1178 1179 static int __init xfrmi_init(void) 1180 { 1181 const char *msg; 1182 int err; 1183 1184 pr_info("IPsec XFRM device driver\n"); 1185 1186 msg = "tunnel device"; 1187 err = register_pernet_device(&xfrmi_net_ops); 1188 if (err < 0) 1189 goto pernet_dev_failed; 1190 1191 msg = "xfrm4 protocols"; 1192 err = xfrmi4_init(); 1193 if (err < 0) 1194 goto xfrmi4_failed; 1195 1196 msg = "xfrm6 protocols"; 1197 err = xfrmi6_init(); 1198 if (err < 0) 1199 goto xfrmi6_failed; 1200 1201 1202 msg = "netlink interface"; 1203 err = rtnl_link_register(&xfrmi_link_ops); 1204 if (err < 0) 1205 goto rtnl_link_failed; 1206 1207 err = register_xfrm_interface_bpf(); 1208 if (err < 0) 1209 goto kfunc_failed; 1210 1211 lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); 1212 1213 xfrm_if_register_cb(&xfrm_if_cb); 1214 1215 return err; 1216 1217 kfunc_failed: 1218 rtnl_link_unregister(&xfrmi_link_ops); 1219 rtnl_link_failed: 1220 xfrmi6_fini(); 1221 xfrmi6_failed: 1222 xfrmi4_fini(); 1223 xfrmi4_failed: 1224 unregister_pernet_device(&xfrmi_net_ops); 1225 pernet_dev_failed: 1226 pr_err("xfrmi init: failed to register %s\n", msg); 1227 return err; 1228 } 1229 1230 static void __exit xfrmi_fini(void) 1231 { 1232 xfrm_if_unregister_cb(); 1233 lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); 1234 rtnl_link_unregister(&xfrmi_link_ops); 1235 xfrmi4_fini(); 1236 xfrmi6_fini(); 1237 unregister_pernet_device(&xfrmi_net_ops); 1238 } 1239 1240 module_init(xfrmi_init); 1241 module_exit(xfrmi_fini); 1242 MODULE_LICENSE("GPL"); 1243 MODULE_ALIAS_RTNL_LINK("xfrm"); 1244 MODULE_ALIAS_NETDEV("xfrm0"); 1245 MODULE_AUTHOR("Steffen Klassert"); 1246 MODULE_DESCRIPTION("XFRM virtual interface"); 1247