Lines Matching +full:hw +full:- +full:gro
1 // SPDX-License-Identifier: GPL-2.0-or-later
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
64 * Pekka Riikonen : Netdev boot-time settings code
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
108 #include <net/gro.h>
169 #include "net-sysfs.h"
189 unsigned int val = net->dev_base_seq + 1; in dev_base_seq_inc()
191 WRITE_ONCE(net->dev_base_seq, val ?: 1); in dev_base_seq_inc()
198 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; in dev_name_hash()
203 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; in dev_index_hash()
235 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); in backlog_lock_irq_save()
243 spin_lock_irq(&sd->input_pkt_queue.lock); in backlog_lock_irq_disable()
252 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags); in backlog_unlock_irq_restore()
260 spin_unlock_irq(&sd->input_pkt_queue.lock); in backlog_unlock_irq_enable()
273 INIT_HLIST_NODE(&name_node->hlist); in netdev_name_node_alloc()
274 name_node->dev = dev; in netdev_name_node_alloc()
275 name_node->name = name; in netdev_name_node_alloc()
284 name_node = netdev_name_node_alloc(dev, dev->name); in netdev_name_node_head_alloc()
287 INIT_LIST_HEAD(&name_node->list); in netdev_name_node_head_alloc()
299 hlist_add_head_rcu(&name_node->hlist, in netdev_name_node_add()
300 dev_name_hash(net, name_node->name)); in netdev_name_node_add()
305 hlist_del_rcu(&name_node->hlist); in netdev_name_node_del()
315 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup()
327 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup_rcu()
345 return -EEXIST; in netdev_name_node_alt_create()
348 return -ENOMEM; in netdev_name_node_alt_create()
350 /* The node that holds dev->name acts as a head of per-device list. */ in netdev_name_node_alt_create()
351 list_add_tail_rcu(&name_node->list, &dev->name_node->list); in netdev_name_node_alt_create()
361 kfree(name_node->name); in netdev_name_node_alt_free()
368 list_del(&name_node->list); in __netdev_name_node_alt_destroy()
369 call_rcu(&name_node->rcu, netdev_name_node_alt_free); in __netdev_name_node_alt_destroy()
379 return -ENOENT; in netdev_name_node_alt_destroy()
383 if (name_node == dev->name_node || name_node->dev != dev) in netdev_name_node_alt_destroy()
384 return -EINVAL; in netdev_name_node_alt_destroy()
394 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { in netdev_name_node_alt_flush()
395 list_del(&name_node->list); in netdev_name_node_alt_flush()
396 netdev_name_node_alt_free(&name_node->rcu); in netdev_name_node_alt_flush()
408 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); in list_netdevice()
409 netdev_name_node_add(net, dev->name_node); in list_netdevice()
410 hlist_add_head_rcu(&dev->index_hlist, in list_netdevice()
411 dev_index_hash(net, dev->ifindex)); in list_netdevice()
417 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); in list_netdevice()
432 xa_erase(&net->dev_by_index, dev->ifindex); in unlist_netdevice()
438 list_del_rcu(&dev->dev_list); in unlist_netdevice()
439 netdev_name_node_del(dev->name_node); in unlist_netdevice()
440 hlist_del_rcu(&dev->index_hlist); in unlist_netdevice()
469 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
470 * according to dev->type
517 return ARRAY_SIZE(netdev_lock_type) - 1; in netdev_lock_pos()
534 i = netdev_lock_pos(dev->type); in netdev_set_addr_lockdep_class()
535 lockdep_set_class_and_name(&dev->addr_list_lock, in netdev_set_addr_lockdep_class()
568 * is cloned and should be copied-on-write, so that it will
570 * --ANK (980803)
575 if (pt->type == htons(ETH_P_ALL)) { in ptype_head()
576 if (!pt->af_packet_net && !pt->dev) in ptype_head()
579 return pt->dev ? &pt->dev->ptype_all : in ptype_head()
580 &pt->af_packet_net->ptype_all; in ptype_head()
583 if (pt->dev) in ptype_head()
584 return &pt->dev->ptype_specific; in ptype_head()
586 return pt->af_packet_net ? &pt->af_packet_net->ptype_specific : in ptype_head()
587 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; in ptype_head()
591 * dev_add_pack - add packet handler
611 list_add_rcu(&pt->list, head); in dev_add_pack()
617 * __dev_remove_pack - remove packet handler
641 list_del_rcu(&pt->list); in __dev_remove_pack()
653 * dev_remove_pack - remove packet handler
680 * dev_get_iflink - get 'iflink' value of a interface
689 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) in dev_get_iflink()
690 return dev->netdev_ops->ndo_get_iflink(dev); in dev_get_iflink()
692 return READ_ONCE(dev->ifindex); in dev_get_iflink()
697 * dev_fill_metadata_dst - Retrieve tunnel egress information.
709 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) in dev_fill_metadata_dst()
710 return -EINVAL; in dev_fill_metadata_dst()
714 return -ENOMEM; in dev_fill_metadata_dst()
715 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) in dev_fill_metadata_dst()
716 return -EINVAL; in dev_fill_metadata_dst()
718 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); in dev_fill_metadata_dst()
724 int k = stack->num_paths++; in dev_fwd_path()
729 return &stack->path[k]; in dev_fwd_path()
743 stack->num_paths = 0; in dev_fill_forward_path()
744 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) { in dev_fill_forward_path()
748 return -1; in dev_fill_forward_path()
751 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path); in dev_fill_forward_path()
753 return -1; in dev_fill_forward_path()
756 return -1; in dev_fill_forward_path()
764 return -1; in dev_fill_forward_path()
765 path->type = DEV_PATH_ETHERNET; in dev_fill_forward_path()
766 path->dev = ctx.dev; in dev_fill_forward_path()
779 if (napi->napi_id == napi_id) in napi_by_id()
795 if (WARN_ON_ONCE(!napi->dev)) in netdev_napi_by_id()
797 if (!net_eq(net, dev_net(napi->dev))) in netdev_napi_by_id()
804 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
822 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { in netdev_napi_by_id_lock()
827 dev = napi->dev; in netdev_napi_by_id_lock()
837 if (napi && napi->dev != dev) in netdev_napi_by_id_lock()
847 * __dev_get_by_name - find a device by its name
863 return node_name ? node_name->dev : NULL; in __dev_get_by_name()
868 * dev_get_by_name_rcu - find a device by its name
884 return node_name ? node_name->dev : NULL; in dev_get_by_name_rcu()
902 * netdev_get_by_name() - find a device by its name
927 * __dev_get_by_index - find a device by its ifindex
943 if (dev->ifindex == ifindex) in __dev_get_by_index()
951 * dev_get_by_index_rcu - find a device by its ifindex
967 if (dev->ifindex == ifindex) in dev_get_by_index_rcu()
988 * netdev_get_by_index() - find a device by its ifindex
1012 * dev_get_by_napi_id - find a device by napi_id
1031 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
1045 if (dev->reg_state > NETREG_REGISTERED) { in __netdev_put_lock()
1055 * netdev_get_by_index_lock() - find a device by its ifindex
1060 * with @ifindex is found it will be returned with netdev->lock held.
1085 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock()
1109 strscpy(name, dev->name, IFNAMSIZ); in netdev_copy_name()
1114 * netdev_get_name - get a netdevice name, knowing its ifindex.
1128 ret = -ENODEV; in netdev_get_name()
1143 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); in dev_addr_cmp()
1147 * dev_getbyhwaddr_rcu - find a device by its hardware address
1174 * dev_getbyhwaddr() - find a device by its hardware address
1205 if (dev->type == type) { in dev_getfirstbyhwtype()
1216 * __dev_get_by_flags - find any device with given flags
1235 if (((dev->flags ^ if_flags) & mask) == 0) { in __dev_get_by_flags()
1245 * dev_valid_name - check if name is okay for network device
1271 * __dev_alloc_name - allocate a name for a device
1276 * Passed a format string - eg "lt%d" it will try and find a suitable
1299 return -EINVAL; in __dev_alloc_name()
1304 return -ENOMEM; in __dev_alloc_name()
1310 if (!sscanf(name_node->name, name, &i)) in __dev_alloc_name()
1317 if (!strncmp(buf, name_node->name, IFNAMSIZ)) in __dev_alloc_name()
1320 if (!sscanf(d->name, name, &i)) in __dev_alloc_name()
1327 if (!strncmp(buf, d->name, IFNAMSIZ)) in __dev_alloc_name()
1334 return -ENFILE; in __dev_alloc_name()
1348 return -EINVAL; in dev_prep_valid_name()
1354 return -dup_errno; in dev_prep_valid_name()
1361 * dev_alloc_name - allocate a name for a device
1365 * Passed a format string - eg "lt%d" it will try and find a suitable
1376 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); in dev_alloc_name()
1385 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); in dev_get_valid_name()
1399 if (!strncmp(newname, dev->name, IFNAMSIZ)) in netif_change_name()
1402 memcpy(oldname, dev->name, IFNAMSIZ); in netif_change_name()
1413 dev->flags & IFF_UP ? " (while UP)" : ""); in netif_change_name()
1415 old_assign_type = dev->name_assign_type; in netif_change_name()
1416 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); in netif_change_name()
1419 ret = device_rename(&dev->dev, dev->name); in netif_change_name()
1422 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1424 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1430 netdev_name_node_del(dev->name_node); in netif_change_name()
1434 netdev_name_node_add(net, dev->name_node); in netif_change_name()
1444 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1447 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1464 return -EINVAL; in netif_set_alias()
1469 return -ENOMEM; in netif_set_alias()
1471 memcpy(new_alias->ifalias, alias, len); in netif_set_alias()
1472 new_alias->ifalias[len] = 0; in netif_set_alias()
1476 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, in netif_set_alias()
1487 * dev_get_alias - get ifalias of a device
1501 alias = rcu_dereference(dev->ifalias); in dev_get_alias()
1503 ret = snprintf(name, len, "%s", alias->ifalias); in dev_get_alias()
1510 * netdev_features_change - device changes features
1525 if (dev->flags & IFF_UP) { in netif_state_change()
1537 * __netdev_notify_peers - notify network peers about existence of @dev,
1556 * netdev_notify_peers - notify network peers about existence of @dev
1583 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", in napi_kthread_create()
1584 n->dev->name, n->napi_id); in napi_kthread_create()
1585 if (IS_ERR(n->thread)) { in napi_kthread_create()
1586 err = PTR_ERR(n->thread); in napi_kthread_create()
1588 n->thread = NULL; in napi_kthread_create()
1596 const struct net_device_ops *ops = dev->netdev_ops; in __dev_open()
1603 /* may be detached because parent is runtime-suspended */ in __dev_open()
1604 if (dev->dev.parent) in __dev_open()
1605 pm_runtime_resume(dev->dev.parent); in __dev_open()
1607 return -ENODEV; in __dev_open()
1621 set_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1625 if (ops->ndo_validate_addr) in __dev_open()
1626 ret = ops->ndo_validate_addr(dev); in __dev_open()
1628 if (!ret && ops->ndo_open) in __dev_open()
1629 ret = ops->ndo_open(dev); in __dev_open()
1634 clear_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1639 add_device_randomness(dev->dev_addr, dev->addr_len); in __dev_open()
1649 if (dev->flags & IFF_UP) in netif_open()
1675 clear_bit(__LINK_STATE_START, &dev->state); in __dev_close_many()
1680 * dev->stop() will invoke napi_disable() on all of it's in __dev_close_many()
1689 const struct net_device_ops *ops = dev->netdev_ops; in __dev_close_many()
1695 * We allow it to be called even after a DETACH hot-plug in __dev_close_many()
1701 if (ops->ndo_stop) in __dev_close_many()
1702 ops->ndo_stop(dev); in __dev_close_many()
1713 list_add(&dev->close_list, &single); in __dev_close()
1724 if (!(dev->flags & IFF_UP)) in dev_close_many()
1725 list_del_init(&dev->close_list); in dev_close_many()
1733 list_del_init(&dev->close_list); in dev_close_many()
1740 if (dev->flags & IFF_UP) { in netif_close()
1743 list_add(&dev->close_list, &single); in netif_close()
1755 dev->wanted_features &= ~NETIF_F_LRO; in netif_disable_lro()
1758 if (unlikely(dev->features & NETIF_F_LRO)) in netif_disable_lro()
1770 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1773 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1779 dev->wanted_features &= ~NETIF_F_GRO_HW; in dev_disable_gro_hw()
1782 if (unlikely(dev->features & NETIF_F_GRO_HW)) in dev_disable_gro_hw()
1817 return nb->notifier_call(nb, val, &info); in call_netdevice_notifier()
1830 if (!(dev->flags & IFF_UP)) in call_netdevice_register_notifiers()
1840 if (dev->flags & IFF_UP) { in call_netdevice_unregister_notifiers()
1881 * register_netdevice_notifier - register a network notifier block
1936 * unregister_netdevice_notifier - unregister a network notifier block
1980 err = raw_notifier_chain_register(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
1993 raw_notifier_chain_unregister(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2002 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); in __unregister_netdevice_notifier_net()
2011 * register_netdevice_notifier_net - register a per-netns network notifier block
2038 * unregister_netdevice_notifier_net - unregister a per-netns
2093 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { in rtnl_net_dev_lock()
2119 nn->nb = nb; in register_netdevice_notifier_dev_net()
2120 list_add(&nn->list, &dev->net_notifier_list); in register_netdevice_notifier_dev_net()
2135 list_del(&nn->list); in unregister_netdevice_notifier_dev_net()
2148 list_for_each_entry(nn, &dev->net_notifier_list, list) in move_netdevice_notifiers_dev_net()
2149 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); in move_netdevice_notifiers_dev_net()
2153 * call_netdevice_notifiers_info - call all network notifier blocks
2164 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info()
2169 /* Run per-netns notifier block chain first, then run the global one. in call_netdevice_notifiers_info()
2171 * all notifier block registrators get converted to be per-netns. in call_netdevice_notifiers_info()
2173 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); in call_netdevice_notifiers_info()
2180 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2187 * Call all per-netns network notifier blocks, but not notifier blocks on
2197 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info_robust()
2201 return raw_notifier_call_chain_robust(&net->netdev_chain, in call_netdevice_notifiers_info_robust()
2218 * call_netdevice_notifiers - call all network notifier blocks
2233 * call_netdevice_notifiers_mtu - call all network notifier blocks
2333 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) in net_disable_timestamp()
2346 skb->tstamp = 0; in net_timestamp_set()
2347 skb->tstamp_type = SKB_CLOCK_REALTIME; in net_timestamp_set()
2349 skb->tstamp = ktime_get_real(); in net_timestamp_set()
2354 if ((COND) && !(SKB)->tstamp) \
2355 (SKB)->tstamp = ktime_get_real(); \
2370 skb->protocol = eth_type_trans(skb, dev); in __dev_forward_skb2()
2384 * dev_forward_skb - loopback an skb to another netif
2417 return -ENOMEM; in deliver_skb()
2418 refcount_inc(&skb->users); in deliver_skb()
2419 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in deliver_skb()
2431 if (ptype->type != type) in deliver_ptype_list_skb()
2442 if (!ptype->af_packet_priv || !skb->sk) in skb_loop_sk()
2445 if (ptype->id_match) in skb_loop_sk()
2446 return ptype->id_match(ptype, skb->sk); in skb_loop_sk()
2447 else if ((struct sock *)ptype->af_packet_priv == skb->sk) in skb_loop_sk()
2454 * dev_nit_active_rcu - return true if any network interface taps are in use
2465 return !list_empty(&dev_net(dev)->ptype_all) || in dev_nit_active_rcu()
2466 !list_empty(&dev->ptype_all); in dev_nit_active_rcu()
2482 ptype_list = &dev_net_rcu(dev)->ptype_all; in dev_queue_xmit_nit()
2485 if (READ_ONCE(ptype->ignore_outgoing)) in dev_queue_xmit_nit()
2489 * they originated from - MvS (miquels@drinkel.ow.org) in dev_queue_xmit_nit()
2495 deliver_skb(skb2, pt_prev, skb->dev); in dev_queue_xmit_nit()
2507 /* skb->nh should be correctly in dev_queue_xmit_nit()
2513 if (skb_network_header(skb2) < skb2->data || in dev_queue_xmit_nit()
2516 ntohs(skb2->protocol), in dev_queue_xmit_nit()
2517 dev->name); in dev_queue_xmit_nit()
2521 skb2->transport_header = skb2->network_header; in dev_queue_xmit_nit()
2522 skb2->pkt_type = PACKET_OUTGOING; in dev_queue_xmit_nit()
2526 if (ptype_list != &dev->ptype_all) { in dev_queue_xmit_nit()
2527 ptype_list = &dev->ptype_all; in dev_queue_xmit_nit()
2533 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); in dev_queue_xmit_nit()
2542 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2557 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netif_setup_tc()
2560 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2562 dev->num_tc = 0; in netif_setup_tc()
2570 tc = &dev->tc_to_txq[q]; in netif_setup_tc()
2571 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2581 if (dev->num_tc) { in netdev_txq_to_tc()
2582 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netdev_txq_to_tc()
2587 if ((txq - tc->offset) < tc->count) in netdev_txq_to_tc()
2591 /* didn't find it, just return -1 to indicate no match */ in netdev_txq_to_tc()
2592 return -1; in netdev_txq_to_tc()
2612 map = xmap_dereference(dev_maps->attr_map[tci]); in remove_xps_queue()
2616 for (pos = map->len; pos--;) { in remove_xps_queue()
2617 if (map->queues[pos] != index) in remove_xps_queue()
2620 if (map->len > 1) { in remove_xps_queue()
2621 map->queues[pos] = map->queues[--map->len]; in remove_xps_queue()
2626 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); in remove_xps_queue()
2627 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in remove_xps_queue()
2639 int num_tc = dev_maps->num_tc; in remove_xps_queue_cpu()
2643 for (tci = cpu * num_tc; num_tc--; tci++) { in remove_xps_queue_cpu()
2646 for (i = count, j = offset; i--; j++) { in remove_xps_queue_cpu()
2665 RCU_INIT_POINTER(dev->xps_maps[type], NULL); in reset_xps_maps()
2677 dev_maps = xmap_dereference(dev->xps_maps[type]); in clean_xps_maps()
2681 for (j = 0; j < dev_maps->nr_ids; j++) in clean_xps_maps()
2687 for (i = offset + (count - 1); count--; i--) in clean_xps_maps()
2713 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); in netif_reset_xps_queues_gt()
2723 for (pos = 0; map && pos < map->len; pos++) { in expand_xps_map()
2724 if (map->queues[pos] != index) in expand_xps_map()
2729 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ in expand_xps_map()
2731 if (pos < map->alloc_len) in expand_xps_map()
2734 alloc_len = map->alloc_len * 2; in expand_xps_map()
2737 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's in expand_xps_map()
2749 new_map->queues[i] = map->queues[i]; in expand_xps_map()
2750 new_map->alloc_len = alloc_len; in expand_xps_map()
2751 new_map->len = pos; in expand_xps_map()
2761 int i, tci = index * dev_maps->num_tc; in xps_copy_dev_maps()
2765 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in xps_copy_dev_maps()
2770 map = xmap_dereference(dev_maps->attr_map[tci]); in xps_copy_dev_maps()
2771 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in xps_copy_dev_maps()
2782 int i, j, tci, numa_node_id = -2; in __netif_set_xps_queue()
2787 WARN_ON_ONCE(index >= dev->num_tx_queues); in __netif_set_xps_queue()
2789 if (dev->num_tc) { in __netif_set_xps_queue()
2791 num_tc = dev->num_tc; in __netif_set_xps_queue()
2793 return -EINVAL; in __netif_set_xps_queue()
2796 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; in __netif_set_xps_queue()
2800 return -EINVAL; in __netif_set_xps_queue()
2805 dev_maps = xmap_dereference(dev->xps_maps[type]); in __netif_set_xps_queue()
2807 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); in __netif_set_xps_queue()
2808 nr_ids = dev->num_rx_queues; in __netif_set_xps_queue()
2820 * setting up now, as dev->num_tc or nr_ids could have been updated in in __netif_set_xps_queue()
2825 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) in __netif_set_xps_queue()
2829 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), in __netif_set_xps_queue()
2835 return -ENOMEM; in __netif_set_xps_queue()
2838 new_dev_maps->nr_ids = nr_ids; in __netif_set_xps_queue()
2839 new_dev_maps->num_tc = num_tc; in __netif_set_xps_queue()
2843 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; in __netif_set_xps_queue()
2849 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in __netif_set_xps_queue()
2868 /* add tx-queue to CPU/rx-queue maps */ in __netif_set_xps_queue()
2873 map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2874 while ((pos < map->len) && (map->queues[pos] != index)) in __netif_set_xps_queue()
2877 if (pos == map->len) in __netif_set_xps_queue()
2878 map->queues[map->len++] = index; in __netif_set_xps_queue()
2881 if (numa_node_id == -2) in __netif_set_xps_queue()
2884 numa_node_id = -1; in __netif_set_xps_queue()
2894 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); in __netif_set_xps_queue()
2900 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2901 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) { in __netif_set_xps_queue()
2902 map = xmap_dereference(dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2907 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2912 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in __netif_set_xps_queue()
2933 /* removes tx-queue from unused CPUs/rx-queues */ in __netif_set_xps_queue()
2934 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2935 tci = j * dev_maps->num_tc; in __netif_set_xps_queue()
2937 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in __netif_set_xps_queue()
2939 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && in __netif_set_xps_queue()
2940 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) in __netif_set_xps_queue()
2963 for (i = num_tc, tci = j * num_tc; i--; tci++) { in __netif_set_xps_queue()
2964 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2966 xmap_dereference(dev_maps->attr_map[tci]) : in __netif_set_xps_queue()
2976 return -ENOMEM; in __netif_set_xps_queue()
2996 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_all_sb_channels()
2999 while (txq-- != &dev->_tx[0]) { in netdev_unbind_all_sb_channels()
3000 if (txq->sb_dev) in netdev_unbind_all_sb_channels()
3001 netdev_unbind_sb_channel(dev, txq->sb_dev); in netdev_unbind_all_sb_channels()
3013 dev->num_tc = 0; in netdev_reset_tc()
3014 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); in netdev_reset_tc()
3015 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); in netdev_reset_tc()
3021 if (tc >= dev->num_tc) in netdev_set_tc_queue()
3022 return -EINVAL; in netdev_set_tc_queue()
3027 dev->tc_to_txq[tc].count = count; in netdev_set_tc_queue()
3028 dev->tc_to_txq[tc].offset = offset; in netdev_set_tc_queue()
3036 return -EINVAL; in netdev_set_num_tc()
3043 dev->num_tc = num_tc; in netdev_set_num_tc()
3051 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_sb_channel()
3056 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); in netdev_unbind_sb_channel()
3057 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); in netdev_unbind_sb_channel()
3059 while (txq-- != &dev->_tx[0]) { in netdev_unbind_sb_channel()
3060 if (txq->sb_dev == sb_dev) in netdev_unbind_sb_channel()
3061 txq->sb_dev = NULL; in netdev_unbind_sb_channel()
3071 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) in netdev_bind_sb_channel_queue()
3072 return -EINVAL; in netdev_bind_sb_channel_queue()
3075 if ((offset + count) > dev->real_num_tx_queues) in netdev_bind_sb_channel_queue()
3076 return -EINVAL; in netdev_bind_sb_channel_queue()
3079 sb_dev->tc_to_txq[tc].count = count; in netdev_bind_sb_channel_queue()
3080 sb_dev->tc_to_txq[tc].offset = offset; in netdev_bind_sb_channel_queue()
3085 while (count--) in netdev_bind_sb_channel_queue()
3086 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; in netdev_bind_sb_channel_queue()
3096 return -ENODEV; in netdev_set_sb_channel()
3098 /* We allow channels 1 - 32767 to be used for subordinate channels. in netdev_set_sb_channel()
3104 return -EINVAL; in netdev_set_sb_channel()
3106 dev->num_tc = -channel; in netdev_set_sb_channel()
3121 disabling = txq < dev->real_num_tx_queues; in netif_set_real_num_tx_queues()
3123 if (txq < 1 || txq > dev->num_tx_queues) in netif_set_real_num_tx_queues()
3124 return -EINVAL; in netif_set_real_num_tx_queues()
3126 if (dev->reg_state == NETREG_REGISTERED || in netif_set_real_num_tx_queues()
3127 dev->reg_state == NETREG_UNREGISTERING) { in netif_set_real_num_tx_queues()
3131 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, in netif_set_real_num_tx_queues()
3136 if (dev->num_tc) in netif_set_real_num_tx_queues()
3143 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3153 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3161 * netif_set_real_num_rx_queues - set actual number of RX queues used
3174 if (rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_rx_queues()
3175 return -EINVAL; in netif_set_real_num_rx_queues()
3177 if (dev->reg_state == NETREG_REGISTERED) { in netif_set_real_num_rx_queues()
3181 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, in netif_set_real_num_rx_queues()
3187 dev->real_num_rx_queues = rxq; in netif_set_real_num_rx_queues()
3193 * netif_set_real_num_queues - set actual number of RX and TX queues used
3204 unsigned int old_rxq = dev->real_num_rx_queues; in netif_set_real_num_queues()
3207 if (txq < 1 || txq > dev->num_tx_queues || in netif_set_real_num_queues()
3208 rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_queues()
3209 return -EINVAL; in netif_set_real_num_queues()
3211 /* Start from increases, so the error path only does decreases - in netif_set_real_num_queues()
3214 if (rxq > dev->real_num_rx_queues) { in netif_set_real_num_queues()
3219 if (txq > dev->real_num_tx_queues) { in netif_set_real_num_queues()
3224 if (rxq < dev->real_num_rx_queues) in netif_set_real_num_queues()
3226 if (txq < dev->real_num_tx_queues) in netif_set_real_num_queues()
3237 * netif_set_tso_max_size() - set the max size of TSO frames supported
3239 * @size: max skb->len of a TSO frame
3241 * Set the limit on the size of TSO super-frames the device can handle.
3247 dev->tso_max_size = min(GSO_MAX_SIZE, size); in netif_set_tso_max_size()
3248 if (size < READ_ONCE(dev->gso_max_size)) in netif_set_tso_max_size()
3250 if (size < READ_ONCE(dev->gso_ipv4_max_size)) in netif_set_tso_max_size()
3256 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3261 * a single TSO super-frame.
3266 dev->tso_max_segs = segs; in netif_set_tso_max_segs()
3267 if (segs < READ_ONCE(dev->gso_max_segs)) in netif_set_tso_max_segs()
3273 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3279 netif_set_tso_max_size(to, from->tso_max_size); in netif_inherit_tso_max()
3280 netif_set_tso_max_segs(to, from->tso_max_segs); in netif_inherit_tso_max()
3285 * netif_get_num_default_rss_queues - default number of RSS queues
3316 q->next_sched = NULL; in __netif_reschedule()
3317 *sd->output_queue_tailp = q; in __netif_reschedule()
3318 sd->output_queue_tailp = &q->next_sched; in __netif_reschedule()
3325 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) in __netif_schedule()
3336 return (struct dev_kfree_skb_cb *)skb->cb; in get_kfree_skb_cb()
3343 struct Qdisc *q = rcu_dereference(txq->qdisc); in netif_schedule_queue()
3353 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { in netif_tx_wake_queue()
3357 q = rcu_dereference(dev_queue->qdisc); in netif_tx_wake_queue()
3371 if (likely(refcount_read(&skb->users) == 1)) { in dev_kfree_skb_irq_reason()
3373 refcount_set(&skb->users, 0); in dev_kfree_skb_irq_reason()
3374 } else if (likely(!refcount_dec_and_test(&skb->users))) { in dev_kfree_skb_irq_reason()
3377 get_kfree_skb_cb(skb)->reason = reason; in dev_kfree_skb_irq_reason()
3379 skb->next = __this_cpu_read(softnet_data.completion_queue); in dev_kfree_skb_irq_reason()
3397 * netif_device_detach - mark device as removed
3404 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_detach()
3412 * netif_device_attach - mark device as attached
3419 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_attach()
3437 u16 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3439 if (dev->num_tc) { in skb_tx_hash()
3440 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); in skb_tx_hash()
3442 qoffset = sb_dev->tc_to_txq[tc].offset; in skb_tx_hash()
3443 qcount = sb_dev->tc_to_txq[tc].count; in skb_tx_hash()
3446 sb_dev->name, qoffset, tc); in skb_tx_hash()
3448 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3456 hash -= qoffset; in skb_tx_hash()
3458 hash -= qcount; in skb_tx_hash()
3468 struct net_device *dev = skb->dev; in skb_warn_bad_offload()
3475 if (dev->dev.parent) in skb_warn_bad_offload()
3476 name = dev_driver_string(dev->dev.parent); in skb_warn_bad_offload()
3482 name, dev ? &dev->features : &null_features, in skb_warn_bad_offload()
3483 skb->sk ? &skb->sk->sk_route_caps : &null_features); in skb_warn_bad_offload()
3495 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_help()
3500 return -EINVAL; in skb_checksum_help()
3504 return -EFAULT; in skb_checksum_help()
3517 ret = -EINVAL; in skb_checksum_help()
3524 csum = skb_checksum(skb, offset, skb->len - offset, 0); in skb_checksum_help()
3526 offset += skb->csum_offset; in skb_checksum_help()
3537 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; in skb_checksum_help()
3539 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_help()
3550 if (skb->ip_summed != CHECKSUM_PARTIAL) in skb_crc32c_csum_help()
3567 ret = -EINVAL; in skb_crc32c_csum_help()
3576 skb->len - start, ~(__u32)0, in skb_crc32c_csum_help()
3578 *(__le32 *)(skb->data + offset) = crc32c_csum; in skb_crc32c_csum_help()
3587 __be16 type = skb->protocol; in skb_network_protocol()
3596 eth = (struct ethhdr *)skb->data; in skb_network_protocol()
3597 type = eth->h_proto; in skb_network_protocol()
3608 netdev_err(dev, "hw csum failure\n"); in do_netdev_rx_csum_fault()
3626 if (!(dev->features & NETIF_F_HIGHDMA)) { in illegal_highdma()
3627 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in illegal_highdma()
3628 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in illegal_highdma()
3648 features &= skb->dev->mpls_features; in net_mpls_features()
3669 if (skb->ip_summed != CHECKSUM_NONE && in harmonize_features()
3673 if (illegal_highdma(skb->dev, skb)) in harmonize_features()
3698 u16 gso_segs = skb_shinfo(skb)->gso_segs; in gso_features_check()
3700 if (gso_segs > READ_ONCE(dev->gso_max_segs)) in gso_features_check()
3703 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) in gso_features_check()
3706 if (!skb_shinfo(skb)->gso_type) { in gso_features_check()
3717 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) in gso_features_check()
3718 features &= ~dev->gso_partial_features; in gso_features_check()
3723 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { in gso_features_check()
3724 struct iphdr *iph = skb->encapsulation ? in gso_features_check()
3727 if (!(iph->frag_off & htons(IP_DF))) in gso_features_check()
3736 struct net_device *dev = skb->dev; in netif_skb_features()
3737 netdev_features_t features = dev->features; in netif_skb_features()
3746 if (skb->encapsulation) in netif_skb_features()
3747 features &= dev->hw_enc_features; in netif_skb_features()
3751 dev->vlan_features | in netif_skb_features()
3755 if (dev->netdev_ops->ndo_features_check) in netif_skb_features()
3756 features &= dev->netdev_ops->ndo_features_check(skb, dev, in netif_skb_features()
3774 len = skb->len; in xmit_one()
3789 struct sk_buff *next = skb->next; in dev_hard_start_xmit()
3794 skb->next = next; in dev_hard_start_xmit()
3814 !vlan_hw_offload_capable(features, skb->vlan_proto)) in validate_xmit_vlan()
3835 switch (skb->csum_offset) { in skb_csum_hwoffload_help()
3882 if (skb->ip_summed == CHECKSUM_PARTIAL) { in validate_xmit_skb()
3883 if (skb->encapsulation) in validate_xmit_skb()
3910 next = skb->next; in validate_xmit_skb_list()
3914 skb->prev = skb; in validate_xmit_skb_list()
3923 tail->next = skb; in validate_xmit_skb_list()
3924 /* If skb was segmented, skb->prev points to in validate_xmit_skb_list()
3927 tail = skb->prev; in validate_xmit_skb_list()
3937 qdisc_skb_cb(skb)->pkt_len = skb->len; in qdisc_pkt_len_init()
3942 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { in qdisc_pkt_len_init()
3943 u16 gso_segs = shinfo->gso_segs; in qdisc_pkt_len_init()
3950 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { in qdisc_pkt_len_init()
3958 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { in qdisc_pkt_len_init()
3966 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { in qdisc_pkt_len_init()
3967 int payload = skb->len - hdr_len; in qdisc_pkt_len_init()
3972 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); in qdisc_pkt_len_init()
3974 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; in qdisc_pkt_len_init()
3984 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; in dev_qdisc_enqueue()
4003 if (q->flags & TCQ_F_NOLOCK) { in __dev_xmit_skb()
4004 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && in __dev_xmit_skb()
4007 * of q->seqlock to protect from racing with requeuing. in __dev_xmit_skb()
4036 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) { in __dev_xmit_skb()
4043 * This permits qdisc->running owner to get the lock more in __dev_xmit_skb()
4052 spin_lock(&q->busylock); in __dev_xmit_skb()
4055 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { in __dev_xmit_skb()
4058 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && in __dev_xmit_skb()
4061 * This is a work-conserving queue; there are no old skbs in __dev_xmit_skb()
4062 * waiting to be sent out; and the qdisc is not running - in __dev_xmit_skb()
4070 spin_unlock(&q->busylock); in __dev_xmit_skb()
4079 WRITE_ONCE(q->owner, smp_processor_id()); in __dev_xmit_skb()
4081 WRITE_ONCE(q->owner, -1); in __dev_xmit_skb()
4084 spin_unlock(&q->busylock); in __dev_xmit_skb()
4096 spin_unlock(&q->busylock); in __dev_xmit_skb()
4107 if (skb->priority) in skb_update_prio()
4109 map = rcu_dereference_bh(skb->dev->priomap); in skb_update_prio()
4116 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); in skb_update_prio()
4118 if (prioidx < map->priomap_len) in skb_update_prio()
4119 skb->priority = map->priomap[prioidx]; in skb_update_prio()
4126 * dev_loopback_xmit - loop back @skb
4135 skb->pkt_type = PACKET_LOOPBACK; in dev_loopback_xmit()
4136 if (skb->ip_summed == CHECKSUM_NONE) in dev_loopback_xmit()
4137 skb->ip_summed = CHECKSUM_UNNECESSARY; in dev_loopback_xmit()
4169 return current->net_xmit.skip_txqueue; in netdev_xmit_txqueue_skipped()
4174 current->net_xmit.skip_txqueue = skip; in netdev_xmit_skip_txqueue()
4186 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); in tc_run()
4196 /* Block-wise bypass */ in tc_run()
4197 if (tcf_block_bypass_sw(miniq->block)) in tc_run()
4200 tc_skb_cb(skb)->mru = 0; in tc_run()
4201 tc_skb_cb(skb)->post_ct = false; in tc_run()
4205 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); in tc_run()
4214 skb->tc_index = TC_H_MIN(res.classid); in tc_run()
4242 __skb_push(skb, skb->mac_len); in tcx_run()
4250 __skb_pull(skb, skb->mac_len); in tcx_run()
4258 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); in sch_handle_ingress()
4272 qdisc_skb_cb(skb)->pkt_len = skb->len; in sch_handle_ingress()
4288 __skb_push(skb, skb->mac_len); in sch_handle_ingress()
4289 if (skb_do_redirect(skb) == -EAGAIN) { in sch_handle_ingress()
4290 __skb_pull(skb, skb->mac_len); in sch_handle_ingress()
4321 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); in sch_handle_egress()
4331 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was in sch_handle_egress()
4387 int tc = netdev_get_prio_tc_map(dev, skb->priority); in __get_xps_queue_idx()
4389 int queue_index = -1; in __get_xps_queue_idx()
4391 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) in __get_xps_queue_idx()
4394 tci *= dev_maps->num_tc; in __get_xps_queue_idx()
4397 map = rcu_dereference(dev_maps->attr_map[tci]); in __get_xps_queue_idx()
4399 if (map->len == 1) in __get_xps_queue_idx()
4400 queue_index = map->queues[0]; in __get_xps_queue_idx()
4402 queue_index = map->queues[reciprocal_scale( in __get_xps_queue_idx()
4403 skb_get_hash(skb), map->len)]; in __get_xps_queue_idx()
4404 if (unlikely(queue_index >= dev->real_num_tx_queues)) in __get_xps_queue_idx()
4405 queue_index = -1; in __get_xps_queue_idx()
4416 struct sock *sk = skb->sk; in get_xps_queue()
4417 int queue_index = -1; in get_xps_queue()
4420 return -1; in get_xps_queue()
4426 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); in get_xps_queue()
4437 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); in get_xps_queue()
4439 unsigned int tci = skb->sender_cpu - 1; in get_xps_queue()
4449 return -1; in get_xps_queue()
4463 struct sock *sk = skb->sk; in netdev_pick_tx()
4468 if (queue_index < 0 || skb->ooo_okay || in netdev_pick_tx()
4469 queue_index >= dev->real_num_tx_queues) { in netdev_pick_tx()
4477 rcu_access_pointer(sk->sk_dst_cache)) in netdev_pick_tx()
4494 u32 sender_cpu = skb->sender_cpu - 1; in netdev_core_pick_tx()
4497 skb->sender_cpu = raw_smp_processor_id() + 1; in netdev_core_pick_tx()
4500 if (dev->real_num_tx_queues != 1) { in netdev_core_pick_tx()
4501 const struct net_device_ops *ops = dev->netdev_ops; in netdev_core_pick_tx()
4503 if (ops->ndo_select_queue) in netdev_core_pick_tx()
4504 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); in netdev_core_pick_tx()
4516 * __dev_queue_xmit() - transmit a buffer
4532 * * 0 - buffer successfully transmitted
4533 * * positive qdisc return code - NET_XMIT_DROP etc.
4534 * * negative errno - other errors
4538 struct net_device *dev = skb->dev; in __dev_queue_xmit()
4541 int rc = -ENOMEM; in __dev_queue_xmit()
4547 if (unlikely(skb_shinfo(skb)->tx_flags & in __dev_queue_xmit()
4549 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); in __dev_queue_xmit()
4580 /* If device/qdisc don't need skb->dst, release it right now while in __dev_queue_xmit()
4583 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) in __dev_queue_xmit()
4591 q = rcu_dereference_bh(txq->qdisc); in __dev_queue_xmit()
4594 if (q->enqueue) { in __dev_queue_xmit()
4611 if (dev->flags & IFF_UP) { in __dev_queue_xmit()
4614 /* Other cpus might concurrently change txq->xmit_lock_owner in __dev_queue_xmit()
4615 * to -1 or to their cpu id, but not to our id. in __dev_queue_xmit()
4617 if (READ_ONCE(txq->xmit_lock_owner) != cpu) { in __dev_queue_xmit()
4638 dev->name); in __dev_queue_xmit()
4645 dev->name); in __dev_queue_xmit()
4649 rc = -ENETDOWN; in __dev_queue_xmit()
4663 struct net_device *dev = skb->dev; in __dev_direct_xmit()
4715 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { in ____napi_schedule()
4719 * read on napi->thread. Only call in ____napi_schedule()
4722 thread = READ_ONCE(napi->thread); in ____napi_schedule()
4727 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in ____napi_schedule()
4734 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
4735 WRITE_ONCE(napi->list_owner, smp_processor_id()); in ____napi_schedule()
4739 if (!sd->in_net_rx_action) in ____napi_schedule()
4752 return hash_32(hash, flow_table->log); in rfs_slot()
4770 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || in set_rps_cpu()
4771 !(dev->features & NETIF_F_NTUPLE)) in set_rps_cpu()
4773 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); in set_rps_cpu()
4777 rxqueue = dev->_rx + rxq_index; in set_rps_cpu()
4778 flow_table = rcu_dereference(rxqueue->rps_flow_table); in set_rps_cpu()
4782 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, in set_rps_cpu()
4787 rflow = &flow_table->flows[flow_id]; in set_rps_cpu()
4788 WRITE_ONCE(rflow->filter, rc); in set_rps_cpu()
4789 if (old_rflow->filter == rc) in set_rps_cpu()
4790 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); in set_rps_cpu()
4794 rps_input_queue_tail_save(&rflow->last_qtail, head); in set_rps_cpu()
4797 WRITE_ONCE(rflow->cpu, next_cpu); in set_rps_cpu()
4810 struct netdev_rx_queue *rxqueue = dev->_rx; in get_rps_cpu()
4813 int cpu = -1; in get_rps_cpu()
4820 if (unlikely(index >= dev->real_num_rx_queues)) { in get_rps_cpu()
4821 WARN_ONCE(dev->real_num_rx_queues > 1, in get_rps_cpu()
4824 dev->name, index, dev->real_num_rx_queues); in get_rps_cpu()
4832 flow_table = rcu_dereference(rxqueue->rps_flow_table); in get_rps_cpu()
4833 map = rcu_dereference(rxqueue->rps_map); in get_rps_cpu()
4851 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]); in get_rps_cpu()
4860 rflow = &flow_table->flows[rfs_slot(hash, flow_table)]; in get_rps_cpu()
4861 tcpu = rflow->cpu; in get_rps_cpu()
4865 * different from current CPU (one in the rx-queue flow in get_rps_cpu()
4867 * - Current CPU is unset (>= nr_cpu_ids). in get_rps_cpu()
4868 * - Current CPU is offline. in get_rps_cpu()
4869 * - The current CPU's queue tail has advanced beyond the in get_rps_cpu()
4876 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - in get_rps_cpu()
4877 rflow->last_qtail)) >= 0)) { in get_rps_cpu()
4892 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; in get_rps_cpu()
4906 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4919 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; in rps_may_expire_flow()
4926 flow_table = rcu_dereference(rxqueue->rps_flow_table); in rps_may_expire_flow()
4927 if (flow_table && flow_id < (1UL << flow_table->log)) { in rps_may_expire_flow()
4928 rflow = &flow_table->flows[flow_id]; in rps_may_expire_flow()
4929 cpu = READ_ONCE(rflow->cpu); in rps_may_expire_flow()
4930 if (READ_ONCE(rflow->filter) == filter_id && cpu < nr_cpu_ids && in rps_may_expire_flow()
4931 ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) - in rps_may_expire_flow()
4932 READ_ONCE(rflow->last_qtail)) < in rps_may_expire_flow()
4933 (int)(10 << flow_table->log))) in rps_may_expire_flow()
4948 ____napi_schedule(sd, &sd->backlog); in rps_trigger_softirq()
4949 sd->received_rps++; in rps_trigger_softirq()
4960 smp_store_release(&sd->defer_ipi_scheduled, 0); in trigger_rx_softirq()
4964 * After we queued a packet into sd->input_pkt_queue,
4967 * - If this is another cpu queue, link it to our rps_ipi_list,
4970 * - If this is our own queue, NAPI schedule our backlog.
4980 __napi_schedule_irqoff(&sd->backlog); in napi_schedule_rps()
4984 sd->rps_ipi_next = mysd->rps_ipi_list; in napi_schedule_rps()
4985 mysd->rps_ipi_list = sd; in napi_schedule_rps()
4990 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) in napi_schedule_rps()
4995 __napi_schedule_irqoff(&mysd->backlog); in napi_schedule_rps()
5005 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) in kick_defer_list_purge()
5006 __napi_schedule_irqoff(&sd->backlog); in kick_defer_list_purge()
5010 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { in kick_defer_list_purge()
5011 smp_call_function_single_async(cpu, &sd->defer_csd); in kick_defer_list_purge()
5032 fl = rcu_dereference(sd->flow_limit); in skb_flow_limit()
5034 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); in skb_flow_limit()
5035 old_flow = fl->history[fl->history_head]; in skb_flow_limit()
5036 fl->history[fl->history_head] = new_flow; in skb_flow_limit()
5038 fl->history_head++; in skb_flow_limit()
5039 fl->history_head &= FLOW_LIMIT_HISTORY - 1; in skb_flow_limit()
5041 if (likely(fl->buckets[old_flow])) in skb_flow_limit()
5042 fl->buckets[old_flow]--; in skb_flow_limit()
5044 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { in skb_flow_limit()
5045 fl->count++; in skb_flow_limit()
5070 if (!netif_running(skb->dev)) in enqueue_to_backlog()
5076 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); in enqueue_to_backlog()
5081 qlen = skb_queue_len(&sd->input_pkt_queue); in enqueue_to_backlog()
5088 &sd->backlog.state)) in enqueue_to_backlog()
5091 __skb_queue_tail(&sd->input_pkt_queue, skb); in enqueue_to_backlog()
5103 atomic_inc(&sd->dropped); in enqueue_to_backlog()
5105 dev_core_stats_rx_dropped_inc(skb->dev); in enqueue_to_backlog()
5112 struct net_device *dev = skb->dev; in netif_get_rxqueue()
5115 rxqueue = dev->_rx; in netif_get_rxqueue()
5120 if (unlikely(index >= dev->real_num_rx_queues)) { in netif_get_rxqueue()
5121 WARN_ONCE(dev->real_num_rx_queues > 1, in netif_get_rxqueue()
5124 dev->name, index, dev->real_num_rx_queues); in netif_get_rxqueue()
5148 mac_len = skb->data - skb_mac_header(skb); in bpf_prog_run_generic_xdp()
5149 hard_start = skb->data - skb_headroom(skb); in bpf_prog_run_generic_xdp()
5152 frame_sz = (void *)skb_end_pointer(skb) - hard_start; in bpf_prog_run_generic_xdp()
5156 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); in bpf_prog_run_generic_xdp()
5157 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, in bpf_prog_run_generic_xdp()
5160 skb_shinfo(skb)->xdp_frags_size = skb->data_len; in bpf_prog_run_generic_xdp()
5166 orig_data_end = xdp->data_end; in bpf_prog_run_generic_xdp()
5167 orig_data = xdp->data; in bpf_prog_run_generic_xdp()
5168 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5169 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); in bpf_prog_run_generic_xdp()
5170 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); in bpf_prog_run_generic_xdp()
5171 orig_eth_type = eth->h_proto; in bpf_prog_run_generic_xdp()
5176 off = xdp->data - orig_data; in bpf_prog_run_generic_xdp()
5181 __skb_push(skb, -off); in bpf_prog_run_generic_xdp()
5183 skb->mac_header += off; in bpf_prog_run_generic_xdp()
5188 off = xdp->data_end - orig_data_end; in bpf_prog_run_generic_xdp()
5190 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); in bpf_prog_run_generic_xdp()
5191 skb->len += off; /* positive on grow, negative on shrink */ in bpf_prog_run_generic_xdp()
5198 skb->data_len = skb_shinfo(skb)->xdp_frags_size; in bpf_prog_run_generic_xdp()
5200 skb->data_len = 0; in bpf_prog_run_generic_xdp()
5203 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5204 if ((orig_eth_type != eth->h_proto) || in bpf_prog_run_generic_xdp()
5205 (orig_host != ether_addr_equal_64bits(eth->h_dest, in bpf_prog_run_generic_xdp()
5206 skb->dev->dev_addr)) || in bpf_prog_run_generic_xdp()
5207 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { in bpf_prog_run_generic_xdp()
5209 skb->pkt_type = PACKET_HOST; in bpf_prog_run_generic_xdp()
5210 skb->protocol = eth_type_trans(skb, skb->dev); in bpf_prog_run_generic_xdp()
5226 metalen = xdp->data - xdp->data_meta; in bpf_prog_run_generic_xdp()
5247 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); in netif_skb_check_for_xdp()
5248 troom = skb->tail + skb->data_len - skb->end; in netif_skb_check_for_xdp()
5275 mac_len = skb->data - skb_mac_header(skb); in netif_receive_generic_xdp()
5293 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5296 trace_xdp_exception((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5308 * network taps in order to match in-driver-XDP behavior. This also means
5310 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5315 struct net_device *dev = skb->dev; in generic_xdp_tx()
5352 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, in do_xdp_generic()
5389 cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_rx_internal()
5393 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_rx_internal()
5407 * __netif_rx - Slightly optimized version of netif_rx
5428 * netif_rx - post buffer to the network code
5436 * driver should use NAPI and GRO.
5466 if (sd->completion_queue) { in net_tx_action()
5470 clist = sd->completion_queue; in net_tx_action()
5471 sd->completion_queue = NULL; in net_tx_action()
5477 clist = clist->next; in net_tx_action()
5479 WARN_ON(refcount_read(&skb->users)); in net_tx_action()
5480 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) in net_tx_action()
5484 get_kfree_skb_cb(skb)->reason, NULL); in net_tx_action()
5486 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) in net_tx_action()
5490 get_kfree_skb_cb(skb)->reason); in net_tx_action()
5494 if (sd->output_queue) { in net_tx_action()
5498 head = sd->output_queue; in net_tx_action()
5499 sd->output_queue = NULL; in net_tx_action()
5500 sd->output_queue_tailp = &sd->output_queue; in net_tx_action()
5509 head = head->next_sched; in net_tx_action()
5511 /* We need to make sure head->next_sched is read in net_tx_action()
5516 if (!(q->flags & TCQ_F_NOLOCK)) { in net_tx_action()
5520 &q->state))) { in net_tx_action()
5529 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5533 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5553 * netdev_is_rx_handler_busy - check if receive handler is registered
5564 return dev && rtnl_dereference(dev->rx_handler); in netdev_is_rx_handler_busy()
5569 * netdev_rx_handler_register - register receive handler
5587 return -EBUSY; in netdev_rx_handler_register()
5589 if (dev->priv_flags & IFF_NO_RX_HANDLER) in netdev_rx_handler_register()
5590 return -EINVAL; in netdev_rx_handler_register()
5593 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); in netdev_rx_handler_register()
5594 rcu_assign_pointer(dev->rx_handler, rx_handler); in netdev_rx_handler_register()
5601 * netdev_rx_handler_unregister - unregister receive handler
5612 RCU_INIT_POINTER(dev->rx_handler, NULL); in netdev_rx_handler_unregister()
5618 RCU_INIT_POINTER(dev->rx_handler_data, NULL); in netdev_rx_handler_unregister()
5628 switch (skb->protocol) { in skb_pfmemalloc_protocol()
5674 orig_dev = skb->dev; in __netif_receive_skb_core()
5690 skb->skb_iif = skb->dev->ifindex; in __netif_receive_skb_core()
5698 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), in __netif_receive_skb_core()
5708 if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5720 list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all, in __netif_receive_skb_core()
5727 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { in __netif_receive_skb_core()
5767 rx_handler = rcu_dereference(skb->dev->rx_handler); in __netif_receive_skb_core()
5789 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { in __netif_receive_skb_core()
5795 skb->pkt_type = PACKET_OTHERHOST; in __netif_receive_skb_core()
5796 } else if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5820 * and set skb->priority like in vlan_do_receive() in __netif_receive_skb_core()
5826 type = skb->protocol; in __netif_receive_skb_core()
5834 /* orig_dev and skb->dev could belong to different netns; in __netif_receive_skb_core()
5836 * coming from skb->dev, as the ptype owner (packet socket) in __netif_receive_skb_core()
5837 * will use dev_net(skb->dev) to do namespace filtering. in __netif_receive_skb_core()
5840 &dev_net_rcu(skb->dev)->ptype_specific); in __netif_receive_skb_core()
5844 &orig_dev->ptype_specific); in __netif_receive_skb_core()
5846 if (unlikely(skb->dev != orig_dev)) { in __netif_receive_skb_core()
5848 &skb->dev->ptype_specific); in __netif_receive_skb_core()
5858 dev_core_stats_rx_dropped_inc(skb->dev); in __netif_receive_skb_core()
5860 dev_core_stats_rx_nohandler_inc(skb->dev); in __netif_receive_skb_core()
5863 * me how you were going to use this. :-) in __netif_receive_skb_core()
5870 * then skb should also be non-NULL. in __netif_receive_skb_core()
5881 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_one_core()
5887 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, in __netif_receive_skb_one_core()
5888 skb->dev, pt_prev, orig_dev); in __netif_receive_skb_one_core()
5893 * netif_receive_skb_core - special purpose version of netif_receive_skb
5929 if (pt_prev->list_func != NULL) in __netif_receive_skb_list_ptype()
5930 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, in __netif_receive_skb_list_ptype()
5935 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in __netif_receive_skb_list_ptype()
5941 /* Fast-path assumptions: in __netif_receive_skb_list_core()
5942 * - There is no RX handler. in __netif_receive_skb_list_core()
5943 * - Only one packet_type matches. in __netif_receive_skb_list_core()
5944 * If either of these fails, we will end up doing some per-packet in __netif_receive_skb_list_core()
5945 * processing in-line, then handling the 'last ptype' for the whole in __netif_receive_skb_list_core()
5946 * sublist. This can't cause out-of-order delivery to any single ptype, in __netif_receive_skb_list_core()
5948 * other ptypes are handled per-packet. in __netif_receive_skb_list_core()
5958 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_list_core()
5973 list_add_tail(&skb->list, &sublist); in __netif_receive_skb_list_core()
5989 * - be delivered to SOCK_MEMALLOC sockets only in __netif_receive_skb()
5990 * - stay away from userspace in __netif_receive_skb()
5991 * - have bounded memory usage in __netif_receive_skb()
6016 list_cut_before(&sublist, head, &skb->list); in __netif_receive_skb_list()
6037 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); in generic_xdp_install()
6038 struct bpf_prog *new = xdp->prog; in generic_xdp_install()
6041 switch (xdp->command) { in generic_xdp_install()
6043 rcu_assign_pointer(dev->xdp_prog, new); in generic_xdp_install()
6057 ret = -EINVAL; in generic_xdp_install()
6077 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_internal()
6080 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_internal()
6101 list_add_tail(&skb->list, &sublist); in netif_receive_skb_list_internal()
6110 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_list_internal()
6115 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_list_internal()
6125 * netif_receive_skb - process receive buffer from network
6153 * netif_receive_skb_list - process many receive buffers from network
6189 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { in flush_backlog()
6190 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6191 __skb_unlink(skb, &sd->input_pkt_queue); in flush_backlog()
6199 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { in flush_backlog()
6200 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6201 __skb_unlink(skb, &sd->process_queue); in flush_backlog()
6223 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || in flush_required()
6224 !skb_queue_empty_lockless(&sd->process_queue); in flush_required()
6231 * input_pkt_queue and process_queue even if the latter could end-up in flush_required()
6260 cpumask_clear(&ptr->flush_cpus); in flush_all_backlogs()
6266 INIT_WORK(&ptr->w[cpu], flush_backlog); in flush_all_backlogs()
6267 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); in flush_all_backlogs()
6268 __cpumask_set_cpu(cpu, &ptr->flush_cpus); in flush_all_backlogs()
6276 for_each_cpu(cpu, &ptr->flush_cpus) in flush_all_backlogs()
6277 flush_work(&ptr->w[cpu]); in flush_all_backlogs()
6291 struct softnet_data *next = remsd->rps_ipi_next; in net_rps_send_ipi()
6293 if (cpu_online(remsd->cpu)) in net_rps_send_ipi()
6294 smp_call_function_single_async(remsd->cpu, &remsd->csd); in net_rps_send_ipi()
6307 struct softnet_data *remsd = sd->rps_ipi_list; in net_rps_action_and_irq_enable()
6310 sd->rps_ipi_list = NULL; in net_rps_action_and_irq_enable()
6324 return !use_backlog_threads() && sd->rps_ipi_list; in sd_has_rps_ipi_waiting()
6344 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); in process_backlog()
6349 while ((skb = __skb_dequeue(&sd->process_queue))) { in process_backlog()
6364 if (skb_queue_empty(&sd->input_pkt_queue)) { in process_backlog()
6373 napi->state &= NAPIF_STATE_THREADED; in process_backlog()
6377 skb_queue_splice_tail_init(&sd->input_pkt_queue, in process_backlog()
6378 &sd->process_queue); in process_backlog()
6390 * __napi_schedule - schedule for receive
6407 * napi_schedule_prep - check if napi can be scheduled
6417 unsigned long new, val = READ_ONCE(n->state); in napi_schedule_prep()
6432 } while (!try_cmpxchg(&n->state, &val, new)); in napi_schedule_prep()
6439 * __napi_schedule_irqoff - schedule for receive
6446 * due to force-threaded interrupts and spinlock substitution.
6468 if (unlikely(n->state & (NAPIF_STATE_NPSVC | in napi_complete_done()
6473 if (n->gro.bitmask) in napi_complete_done()
6475 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); in napi_complete_done()
6477 if (n->defer_hard_irqs_count > 0) { in napi_complete_done()
6478 n->defer_hard_irqs_count--; in napi_complete_done()
6487 * the GRO layer. in napi_complete_done()
6489 gro_flush(&n->gro, !!timeout); in napi_complete_done()
6490 gro_normal_list(&n->gro); in napi_complete_done()
6492 if (unlikely(!list_empty(&n->poll_list))) { in napi_complete_done()
6493 /* If n->poll_list is not empty, we need to mask irqs */ in napi_complete_done()
6495 list_del_init(&n->poll_list); in napi_complete_done()
6498 WRITE_ONCE(n->list_owner, -1); in napi_complete_done()
6500 val = READ_ONCE(n->state); in napi_complete_done()
6509 * because we will call napi->poll() one more time. in napi_complete_done()
6514 } while (!try_cmpxchg(&n->state, &val, new)); in napi_complete_done()
6522 hrtimer_start(&n->timer, ns_to_ktime(timeout), in napi_complete_done()
6533 if (!READ_ONCE(sd->defer_list)) in skb_defer_free_flush()
6536 spin_lock(&sd->defer_lock); in skb_defer_free_flush()
6537 skb = sd->defer_list; in skb_defer_free_flush()
6538 sd->defer_list = NULL; in skb_defer_free_flush()
6539 sd->defer_count = 0; in skb_defer_free_flush()
6540 spin_unlock(&sd->defer_lock); in skb_defer_free_flush()
6543 next = skb->next; in skb_defer_free_flush()
6554 gro_normal_list(&napi->gro); in __busy_poll_stop()
6560 gro_flush(&napi->gro, HZ >= 1000); in __busy_poll_stop()
6561 gro_normal_list(&napi->gro); in __busy_poll_stop()
6563 clear_bit(NAPI_STATE_SCHED, &napi->state); in __busy_poll_stop()
6582 * Since we are about to call napi->poll() once more, we can safely in busy_poll_stop()
6588 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6589 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6595 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); in busy_poll_stop()
6597 if (napi->defer_hard_irqs_count && timeout) { in busy_poll_stop()
6598 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); in busy_poll_stop()
6603 /* All we really want here is to re-enable device interrupts. in busy_poll_stop()
6606 rc = napi->poll(napi, budget); in busy_poll_stop()
6607 /* We can't gro_normal_list() here, because napi->poll() might have in busy_poll_stop()
6646 unsigned long val = READ_ONCE(napi->state); in __napi_busy_loop()
6649 * we avoid dirtying napi->state as much as we can. in __napi_busy_loop()
6654 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6657 if (cmpxchg(&napi->state, val, in __napi_busy_loop()
6661 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6665 napi_poll = napi->poll; in __napi_busy_loop()
6669 gro_normal_list(&napi->gro); in __napi_busy_loop()
6672 __NET_ADD_STATS(dev_net(napi->dev), in __napi_busy_loop()
6737 hrtimer_start(&napi->timer, ns_to_ktime(timeout), in napi_suspend_irqs()
6769 napi->gro.cached_napi_id = napi_id; in __napi_hash_add_with_id()
6771 WRITE_ONCE(napi->napi_id, napi_id); in __napi_hash_add_with_id()
6772 hlist_add_head_rcu(&napi->napi_hash_node, in __napi_hash_add_with_id()
6773 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in __napi_hash_add_with_id()
6791 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) in napi_hash_add()
6816 hlist_del_init_rcu(&napi->napi_hash_node); in napi_hash_del()
6831 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { in napi_watchdog()
6832 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_watchdog()
6846 if (dev->threaded == threaded) in dev_set_threaded()
6850 list_for_each_entry(napi, &dev->napi_list, dev_list) { in dev_set_threaded()
6851 if (!napi->thread) { in dev_set_threaded()
6861 WRITE_ONCE(dev->threaded, threaded); in dev_set_threaded()
6874 list_for_each_entry(napi, &dev->napi_list, dev_list) in dev_set_threaded()
6875 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); in dev_set_threaded()
6882 * netif_queue_set_napi - Associate queue with the napi
6889 * registering the NAPI handler for the queue-vector and the queues have been
6898 if (WARN_ON_ONCE(napi && !napi->dev)) in netif_queue_set_napi()
6905 rxq->napi = napi; in netif_queue_set_napi()
6909 txq->napi = napi; in netif_queue_set_napi()
6924 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_irq_notify()
6928 if (napi->config && napi->dev->irq_affinity_auto) in netif_napi_irq_notify()
6929 cpumask_copy(&napi->config->affinity_mask, mask); in netif_napi_irq_notify()
6932 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_irq_notify()
6933 err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask); in netif_napi_irq_notify()
6935 netdev_warn(napi->dev, "RMAP update failed (%d)\n", in netif_napi_irq_notify()
6946 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_affinity_release()
6948 netdev_assert_locked(napi->dev); in netif_napi_affinity_release()
6950 &napi->state)); in netif_napi_affinity_release()
6952 if (!napi->dev->rx_cpu_rmap_auto) in netif_napi_affinity_release()
6954 rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_affinity_release()
6955 napi->napi_rmap_idx = -1; in netif_napi_affinity_release()
6961 if (dev->rx_cpu_rmap_auto) in netif_enable_cpu_rmap()
6964 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs); in netif_enable_cpu_rmap()
6965 if (!dev->rx_cpu_rmap) in netif_enable_cpu_rmap()
6966 return -ENOMEM; in netif_enable_cpu_rmap()
6968 dev->rx_cpu_rmap_auto = true; in netif_enable_cpu_rmap()
6975 struct cpu_rmap *rmap = dev->rx_cpu_rmap; in netif_del_cpu_rmap()
6977 if (!dev->rx_cpu_rmap_auto) in netif_del_cpu_rmap()
6982 dev->rx_cpu_rmap = NULL; in netif_del_cpu_rmap()
6983 dev->rx_cpu_rmap_auto = false; in netif_del_cpu_rmap()
7006 maxqs = max(dev->num_tx_queues, dev->num_rx_queues); in netif_set_affinity_auto()
7007 numa = dev_to_node(&dev->dev); in netif_set_affinity_auto()
7011 &dev->napi_config[i].affinity_mask); in netif_set_affinity_auto()
7013 dev->irq_affinity_auto = true; in netif_set_affinity_auto()
7021 netdev_assert_locked_or_invisible(napi->dev); in netif_napi_set_irq_locked()
7023 if (napi->irq == irq) in netif_napi_set_irq_locked()
7027 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in netif_napi_set_irq_locked()
7028 irq_set_affinity_notifier(napi->irq, NULL); in netif_napi_set_irq_locked()
7030 napi->irq = irq; in netif_napi_set_irq_locked()
7032 (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto)) in netif_napi_set_irq_locked()
7036 if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config)) in netif_napi_set_irq_locked()
7040 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7041 rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi); in netif_napi_set_irq_locked()
7045 cpu_rmap_get(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7046 napi->napi_rmap_idx = rc; in netif_napi_set_irq_locked()
7051 napi->notify.notify = netif_napi_irq_notify; in netif_napi_set_irq_locked()
7052 napi->notify.release = netif_napi_affinity_release; in netif_napi_set_irq_locked()
7053 rc = irq_set_affinity_notifier(irq, &napi->notify); in netif_napi_set_irq_locked()
7055 netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n", in netif_napi_set_irq_locked()
7060 set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state); in netif_napi_set_irq_locked()
7065 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7066 napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_set_irq_locked()
7067 cpu_rmap_put(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7068 napi->napi_rmap_idx = -1; in netif_napi_set_irq_locked()
7071 napi->notify.notify = NULL; in netif_napi_set_irq_locked()
7072 napi->notify.release = NULL; in netif_napi_set_irq_locked()
7078 n->defer_hard_irqs = n->config->defer_hard_irqs; in napi_restore_config()
7079 n->gro_flush_timeout = n->config->gro_flush_timeout; in napi_restore_config()
7080 n->irq_suspend_timeout = n->config->irq_suspend_timeout; in napi_restore_config()
7082 if (n->dev->irq_affinity_auto && in napi_restore_config()
7083 test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state)) in napi_restore_config()
7084 irq_set_affinity(n->irq, &n->config->affinity_mask); in napi_restore_config()
7089 if (n->config->napi_id) { in napi_restore_config()
7090 napi_hash_add_with_id(n, n->config->napi_id); in napi_restore_config()
7093 n->config->napi_id = n->napi_id; in napi_restore_config()
7099 n->config->defer_hard_irqs = n->defer_hard_irqs; in napi_save_config()
7100 n->config->gro_flush_timeout = n->gro_flush_timeout; in napi_save_config()
7101 n->config->irq_suspend_timeout = n->irq_suspend_timeout; in napi_save_config()
7116 if (napi->config && napi->config->napi_id) in netif_napi_dev_list_add()
7117 new_id = napi->config->napi_id; in netif_napi_dev_list_add()
7119 higher = &dev->napi_list; in netif_napi_dev_list_add()
7120 list_for_each_entry(pos, &dev->napi_list, dev_list) { in netif_napi_dev_list_add()
7121 if (napi_id_valid(pos->napi_id)) in netif_napi_dev_list_add()
7122 pos_id = pos->napi_id; in netif_napi_dev_list_add()
7123 else if (pos->config) in netif_napi_dev_list_add()
7124 pos_id = pos->config->napi_id; in netif_napi_dev_list_add()
7130 higher = &pos->dev_list; in netif_napi_dev_list_add()
7132 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ in netif_napi_dev_list_add()
7136 * skb->head being backed by slab, not a page fragment.
7138 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
7147 WARN_ON_ONCE(skb && skb->head_frag); in napi_get_frags_check()
7158 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) in netif_napi_add_weight_locked()
7161 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add_weight_locked()
7162 INIT_HLIST_NODE(&napi->napi_hash_node); in netif_napi_add_weight_locked()
7163 hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add_weight_locked()
7164 gro_init(&napi->gro); in netif_napi_add_weight_locked()
7165 napi->skb = NULL; in netif_napi_add_weight_locked()
7166 napi->poll = poll; in netif_napi_add_weight_locked()
7170 napi->weight = weight; in netif_napi_add_weight_locked()
7171 napi->dev = dev; in netif_napi_add_weight_locked()
7173 napi->poll_owner = -1; in netif_napi_add_weight_locked()
7175 napi->list_owner = -1; in netif_napi_add_weight_locked()
7176 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add_weight_locked()
7177 set_bit(NAPI_STATE_NPSVC, &napi->state); in netif_napi_add_weight_locked()
7180 /* default settings from sysfs are applied to all NAPIs. any per-NAPI in netif_napi_add_weight_locked()
7183 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); in netif_napi_add_weight_locked()
7184 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); in netif_napi_add_weight_locked()
7187 /* Create kthread for this napi if dev->threaded is set. in netif_napi_add_weight_locked()
7188 * Clear dev->threaded if kthread creation failed so that in netif_napi_add_weight_locked()
7191 if (dev->threaded && napi_kthread_create(napi)) in netif_napi_add_weight_locked()
7192 dev->threaded = false; in netif_napi_add_weight_locked()
7193 netif_napi_set_irq_locked(napi, -1); in netif_napi_add_weight_locked()
7202 netdev_assert_locked(n->dev); in napi_disable_locked()
7204 set_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7206 val = READ_ONCE(n->state); in napi_disable_locked()
7210 val = READ_ONCE(n->state); in napi_disable_locked()
7215 } while (!try_cmpxchg(&n->state, &val, new)); in napi_disable_locked()
7217 hrtimer_cancel(&n->timer); in napi_disable_locked()
7219 if (n->config) in napi_disable_locked()
7224 clear_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7229 * napi_disable() - prevent NAPI from scheduling
7238 netdev_lock(n->dev); in napi_disable()
7240 netdev_unlock(n->dev); in napi_disable()
7246 unsigned long new, val = READ_ONCE(n->state); in napi_enable_locked()
7248 if (n->config) in napi_enable_locked()
7257 if (n->dev->threaded && n->thread) in napi_enable_locked()
7259 } while (!try_cmpxchg(&n->state, &val, new)); in napi_enable_locked()
7264 * napi_enable() - enable NAPI scheduling
7273 netdev_lock(n->dev); in napi_enable()
7275 netdev_unlock(n->dev); in napi_enable()
7282 netdev_assert_locked(napi->dev); in __netif_napi_del_locked()
7284 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) in __netif_napi_del_locked()
7288 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); in __netif_napi_del_locked()
7290 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in __netif_napi_del_locked()
7291 irq_set_affinity_notifier(napi->irq, NULL); in __netif_napi_del_locked()
7293 if (napi->config) { in __netif_napi_del_locked()
7294 napi->index = -1; in __netif_napi_del_locked()
7295 napi->config = NULL; in __netif_napi_del_locked()
7298 list_del_rcu(&napi->dev_list); in __netif_napi_del_locked()
7301 gro_cleanup(&napi->gro); in __netif_napi_del_locked()
7303 if (napi->thread) { in __netif_napi_del_locked()
7304 kthread_stop(napi->thread); in __netif_napi_del_locked()
7305 napi->thread = NULL; in __netif_napi_del_locked()
7314 weight = n->weight; in __napi_poll()
7319 * actually make the ->poll() call. Therefore we avoid in __napi_poll()
7320 * accidentally calling ->poll() when NAPI is not scheduled. in __napi_poll()
7324 work = n->poll(n, weight); in __napi_poll()
7331 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", in __napi_poll()
7332 n->poll, work, weight); in __napi_poll()
7340 * move the instance around on the list at-will. in __napi_poll()
7347 /* The NAPI context has more processing work, but busy-polling in __napi_poll()
7353 * that the NAPI is re-scheduled. in __napi_poll()
7361 gro_flush(&n->gro, HZ >= 1000); in __napi_poll()
7362 gro_normal_list(&n->gro); in __napi_poll()
7367 if (unlikely(!list_empty(&n->poll_list))) { in __napi_poll()
7369 n->dev ? n->dev->name : "backlog"); in __napi_poll()
7384 list_del_init(&n->poll_list); in napi_poll()
7391 list_add_tail(&n->poll_list, repoll); in napi_poll()
7408 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { in napi_thread_wait()
7409 WARN_ON(!list_empty(&napi->poll_list)); in napi_thread_wait()
7419 return -1; in napi_thread_wait()
7436 sd->in_napi_threaded_poll = true; in napi_threaded_poll_loop()
7442 sd->in_napi_threaded_poll = false; in napi_threaded_poll_loop()
7483 sd->in_net_rx_action = true; in net_rx_action()
7485 list_splice_init(&sd->poll_list, &list); in net_rx_action()
7495 sd->in_net_rx_action = false; in net_rx_action()
7499 * sd->in_net_rx_action was true. in net_rx_action()
7501 if (!list_empty(&sd->poll_list)) in net_rx_action()
7510 budget -= napi_poll(n, &repoll); in net_rx_action()
7518 sd->time_squeeze++; in net_rx_action()
7525 list_splice_tail_init(&sd->poll_list, &list); in net_rx_action()
7527 list_splice(&list, &sd->poll_list); in net_rx_action()
7528 if (!list_empty(&sd->poll_list)) in net_rx_action()
7531 sd->in_net_rx_action = false; in net_rx_action()
7564 if (adj->dev == adj_dev) in __netdev_find_adj()
7573 struct net_device *dev = (struct net_device *)priv->data; in ____netdev_has_upper_dev()
7579 * netdev_has_upper_dev - Check if device is linked to an upper device
7602 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7624 * netdev_has_any_upper_dev - Check if device is linked to some device
7634 return !list_empty(&dev->adj_list.upper); in netdev_has_any_upper_dev()
7639 * netdev_master_upper_dev_get - Get master upper device
7651 if (list_empty(&dev->adj_list.upper)) in netdev_master_upper_dev_get()
7654 upper = list_first_entry(&dev->adj_list.upper, in netdev_master_upper_dev_get()
7656 if (likely(upper->master)) in netdev_master_upper_dev_get()
7657 return upper->dev; in netdev_master_upper_dev_get()
7668 if (list_empty(&dev->adj_list.upper)) in __netdev_master_upper_dev_get()
7671 upper = list_first_entry(&dev->adj_list.upper, in __netdev_master_upper_dev_get()
7673 if (likely(upper->master) && !upper->ignore) in __netdev_master_upper_dev_get()
7674 return upper->dev; in __netdev_master_upper_dev_get()
7679 * netdev_has_any_lower_dev - Check if device is linked to some device
7689 return !list_empty(&dev->adj_list.lower); in netdev_has_any_lower_dev()
7698 return adj->private; in netdev_adjacent_get_private()
7703 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7717 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_upper_get_next_dev_rcu()
7719 if (&upper->list == &dev->adj_list.upper) in netdev_upper_get_next_dev_rcu()
7722 *iter = &upper->list; in netdev_upper_get_next_dev_rcu()
7724 return upper->dev; in netdev_upper_get_next_dev_rcu()
7734 upper = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_upper_dev()
7736 if (&upper->list == &dev->adj_list.upper) in __netdev_next_upper_dev()
7739 *iter = &upper->list; in __netdev_next_upper_dev()
7740 *ignore = upper->ignore; in __netdev_next_upper_dev()
7742 return upper->dev; in __netdev_next_upper_dev()
7752 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_upper_dev_rcu()
7754 if (&upper->list == &dev->adj_list.upper) in netdev_next_upper_dev_rcu()
7757 *iter = &upper->list; in netdev_next_upper_dev_rcu()
7759 return upper->dev; in netdev_next_upper_dev_rcu()
7773 iter = &dev->adj_list.upper; in __netdev_walk_all_upper_dev()
7791 niter = &udev->adj_list.upper; in __netdev_walk_all_upper_dev()
7800 next = dev_stack[--cur]; in __netdev_walk_all_upper_dev()
7821 iter = &dev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7837 niter = &udev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7846 next = dev_stack[--cur]; in netdev_walk_all_upper_dev_rcu()
7873 * netdev_lower_get_next_private - Get the next ->private from the
7878 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7890 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private()
7893 *iter = lower->list.next; in netdev_lower_get_next_private()
7895 return lower->private; in netdev_lower_get_next_private()
7900 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7906 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7916 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_lower_get_next_private_rcu()
7918 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private_rcu()
7921 *iter = &lower->list; in netdev_lower_get_next_private_rcu()
7923 return lower->private; in netdev_lower_get_next_private_rcu()
7928 * netdev_lower_get_next - Get the next device from the lower neighbour
7944 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next()
7947 *iter = lower->list.next; in netdev_lower_get_next()
7949 return lower->dev; in netdev_lower_get_next()
7958 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev()
7960 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev()
7963 *iter = &lower->list; in netdev_next_lower_dev()
7965 return lower->dev; in netdev_next_lower_dev()
7974 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_lower_dev()
7976 if (&lower->list == &dev->adj_list.lower) in __netdev_next_lower_dev()
7979 *iter = &lower->list; in __netdev_next_lower_dev()
7980 *ignore = lower->ignore; in __netdev_next_lower_dev()
7982 return lower->dev; in __netdev_next_lower_dev()
7995 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev()
8011 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev()
8020 next = dev_stack[--cur]; in netdev_walk_all_lower_dev()
8043 iter = &dev->adj_list.lower; in __netdev_walk_all_lower_dev()
8061 niter = &ldev->adj_list.lower; in __netdev_walk_all_lower_dev()
8070 next = dev_stack[--cur]; in __netdev_walk_all_lower_dev()
8086 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev_rcu()
8087 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev_rcu()
8090 *iter = &lower->list; in netdev_next_lower_dev_rcu()
8092 return lower->dev; in netdev_next_lower_dev_rcu()
8103 for (iter = &dev->adj_list.upper, in __netdev_upper_depth()
8109 if (max_depth < udev->upper_level) in __netdev_upper_depth()
8110 max_depth = udev->upper_level; in __netdev_upper_depth()
8123 for (iter = &dev->adj_list.lower, in __netdev_lower_depth()
8129 if (max_depth < ldev->lower_level) in __netdev_lower_depth()
8130 max_depth = ldev->lower_level; in __netdev_lower_depth()
8139 dev->upper_level = __netdev_upper_depth(dev) + 1; in __netdev_update_upper_level()
8148 if (list_empty(&dev->unlink_list)) in net_unlink_todo()
8149 list_add_tail(&dev->unlink_list, &net_unlink_list); in net_unlink_todo()
8156 dev->lower_level = __netdev_lower_depth(dev) + 1; in __netdev_update_lower_level()
8162 if (priv->flags & NESTED_SYNC_IMM) in __netdev_update_lower_level()
8163 dev->nested_level = dev->lower_level - 1; in __netdev_update_lower_level()
8164 if (priv->flags & NESTED_SYNC_TODO) in __netdev_update_lower_level()
8180 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8196 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8205 next = dev_stack[--cur]; in netdev_walk_all_lower_dev_rcu()
8218 * netdev_lower_get_first_private_rcu - Get the first ->private from the
8223 * Gets the first netdev_adjacent->private from the dev's lower neighbour
8230 lower = list_first_or_null_rcu(&dev->adj_list.lower, in netdev_lower_get_first_private_rcu()
8233 return lower->private; in netdev_lower_get_first_private_rcu()
8239 * netdev_master_upper_dev_get_rcu - Get master upper device
8249 upper = list_first_or_null_rcu(&dev->adj_list.upper, in netdev_master_upper_dev_get_rcu()
8251 if (upper && likely(upper->master)) in netdev_master_upper_dev_get_rcu()
8252 return upper->dev; in netdev_master_upper_dev_get_rcu()
8263 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_add()
8264 "upper_%s" : "lower_%s", adj_dev->name); in netdev_adjacent_sysfs_add()
8265 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), in netdev_adjacent_sysfs_add()
8274 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_del()
8276 sysfs_remove_link(&(dev->dev.kobj), linkname); in netdev_adjacent_sysfs_del()
8283 return (dev_list == &dev->adj_list.upper || in netdev_adjacent_is_neigh_list()
8284 dev_list == &dev->adj_list.lower) && in netdev_adjacent_is_neigh_list()
8299 adj->ref_nr += 1; in __netdev_adjacent_dev_insert()
8300 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", in __netdev_adjacent_dev_insert()
8301 dev->name, adj_dev->name, adj->ref_nr); in __netdev_adjacent_dev_insert()
8308 return -ENOMEM; in __netdev_adjacent_dev_insert()
8310 adj->dev = adj_dev; in __netdev_adjacent_dev_insert()
8311 adj->master = master; in __netdev_adjacent_dev_insert()
8312 adj->ref_nr = 1; in __netdev_adjacent_dev_insert()
8313 adj->private = private; in __netdev_adjacent_dev_insert()
8314 adj->ignore = false; in __netdev_adjacent_dev_insert()
8315 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); in __netdev_adjacent_dev_insert()
8317 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", in __netdev_adjacent_dev_insert()
8318 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); in __netdev_adjacent_dev_insert()
8328 ret = sysfs_create_link(&(dev->dev.kobj), in __netdev_adjacent_dev_insert()
8329 &(adj_dev->dev.kobj), "master"); in __netdev_adjacent_dev_insert()
8333 list_add_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8335 list_add_tail_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8342 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_insert()
8344 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_insert()
8358 dev->name, adj_dev->name, ref_nr); in __netdev_adjacent_dev_remove()
8364 dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8369 if (adj->ref_nr > ref_nr) { in __netdev_adjacent_dev_remove()
8370 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", in __netdev_adjacent_dev_remove()
8371 dev->name, adj_dev->name, ref_nr, in __netdev_adjacent_dev_remove()
8372 adj->ref_nr - ref_nr); in __netdev_adjacent_dev_remove()
8373 adj->ref_nr -= ref_nr; in __netdev_adjacent_dev_remove()
8377 if (adj->master) in __netdev_adjacent_dev_remove()
8378 sysfs_remove_link(&(dev->dev.kobj), "master"); in __netdev_adjacent_dev_remove()
8381 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_remove()
8383 list_del_rcu(&adj->list); in __netdev_adjacent_dev_remove()
8385 adj_dev->name, dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8386 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_remove()
8428 &dev->adj_list.upper, in __netdev_adjacent_dev_link_neighbour()
8429 &upper_dev->adj_list.lower, in __netdev_adjacent_dev_link_neighbour()
8437 &dev->adj_list.upper, in __netdev_adjacent_dev_unlink_neighbour()
8438 &upper_dev->adj_list.lower); in __netdev_adjacent_dev_unlink_neighbour()
8463 return -EBUSY; in __netdev_upper_dev_link()
8467 return -EBUSY; in __netdev_upper_dev_link()
8469 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) in __netdev_upper_dev_link()
8470 return -EMLINK; in __netdev_upper_dev_link()
8474 return -EEXIST; in __netdev_upper_dev_link()
8478 return master_dev == upper_dev ? -EEXIST : -EBUSY; in __netdev_upper_dev_link()
8514 * netdev_upper_dev_link - Add a link to the upper device
8539 * netdev_master_upper_dev_link - Add a master link to the upper device
8547 * one master upper device can be linked, although other non-master devices
8600 * netdev_upper_dev_unlink - Removes a link to upper device
8625 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); in __netdev_adjacent_dev_set()
8627 adj->ignore = val; in __netdev_adjacent_dev_set()
8629 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); in __netdev_adjacent_dev_set()
8631 adj->ignore = val; in __netdev_adjacent_dev_set()
8714 * netdev_bonding_info_change - Dispatch event about slave change
8746 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3), in netdev_offload_xstats_enable_l3()
8748 if (!dev->offload_xstats_l3) in netdev_offload_xstats_enable_l3()
8749 return -ENOMEM; in netdev_offload_xstats_enable_l3()
8761 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_enable_l3()
8762 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_enable_l3()
8773 return -EALREADY; in netdev_offload_xstats_enable()
8781 return -EINVAL; in netdev_offload_xstats_enable()
8794 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_disable_l3()
8795 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_disable_l3()
8804 return -EALREADY; in netdev_offload_xstats_disable()
8813 return -EINVAL; in netdev_offload_xstats_disable()
8828 return dev->offload_xstats_l3; in netdev_offload_xstats_get_ptr()
8856 dest->rx_packets += src->rx_packets; in netdev_hw_stats64_add()
8857 dest->tx_packets += src->tx_packets; in netdev_hw_stats64_add()
8858 dest->rx_bytes += src->rx_bytes; in netdev_hw_stats64_add()
8859 dest->tx_bytes += src->tx_bytes; in netdev_hw_stats64_add()
8860 dest->rx_errors += src->rx_errors; in netdev_hw_stats64_add()
8861 dest->tx_errors += src->tx_errors; in netdev_hw_stats64_add()
8862 dest->rx_dropped += src->rx_dropped; in netdev_hw_stats64_add()
8863 dest->tx_dropped += src->tx_dropped; in netdev_hw_stats64_add()
8864 dest->multicast += src->multicast; in netdev_hw_stats64_add()
8906 return -EINVAL; in netdev_offload_xstats_get_stats()
8943 report_delta->used = true; in netdev_offload_xstats_report_delta()
8944 netdev_hw_stats64_add(&report_delta->stats, stats); in netdev_offload_xstats_report_delta()
8951 report_used->used = true; in netdev_offload_xstats_report_used()
8972 * netdev_get_xmit_slave - Get the xmit slave of master device
8986 const struct net_device_ops *ops = dev->netdev_ops; in netdev_get_xmit_slave()
8988 if (!ops->ndo_get_xmit_slave) in netdev_get_xmit_slave()
8990 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); in netdev_get_xmit_slave()
8997 const struct net_device_ops *ops = dev->netdev_ops; in netdev_sk_get_lower_dev()
8999 if (!ops->ndo_sk_get_lower_dev) in netdev_sk_get_lower_dev()
9001 return ops->ndo_sk_get_lower_dev(dev, sk); in netdev_sk_get_lower_dev()
9005 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
9033 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_add_links()
9034 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9036 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9037 &iter->dev->adj_list.lower); in netdev_adjacent_add_links()
9038 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9039 &dev->adj_list.upper); in netdev_adjacent_add_links()
9042 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_add_links()
9043 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9045 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9046 &iter->dev->adj_list.upper); in netdev_adjacent_add_links()
9047 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9048 &dev->adj_list.lower); in netdev_adjacent_add_links()
9058 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_del_links()
9059 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9061 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9062 &iter->dev->adj_list.lower); in netdev_adjacent_del_links()
9063 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9064 &dev->adj_list.upper); in netdev_adjacent_del_links()
9067 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_del_links()
9068 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9070 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9071 &iter->dev->adj_list.upper); in netdev_adjacent_del_links()
9072 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9073 &dev->adj_list.lower); in netdev_adjacent_del_links()
9083 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_rename_links()
9084 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9086 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9087 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9088 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9089 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9092 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_rename_links()
9093 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9095 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9096 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9097 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9098 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9109 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); in netdev_lower_dev_get_private()
9113 return lower->private; in netdev_lower_dev_get_private()
9119 * netdev_lower_state_changed - Dispatch event about lower device state change
9142 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_rx_flags()
9144 if (ops->ndo_change_rx_flags) in dev_change_rx_flags()
9145 ops->ndo_change_rx_flags(dev, flags); in dev_change_rx_flags()
9150 unsigned int old_flags = dev->flags; in __dev_set_promiscuity()
9157 promiscuity = dev->promiscuity + inc; in __dev_set_promiscuity()
9165 return -EOVERFLOW; in __dev_set_promiscuity()
9171 WRITE_ONCE(dev->promiscuity, promiscuity); in __dev_set_promiscuity()
9173 WRITE_ONCE(dev->flags, flags); in __dev_set_promiscuity()
9175 dev->flags & IFF_PROMISC ? "entered" : "left"); in __dev_set_promiscuity()
9181 dev->name, (dev->flags & IFF_PROMISC), in __dev_set_promiscuity()
9198 unsigned int old_flags = dev->flags; in netif_set_promiscuity()
9204 if (dev->flags != old_flags) in netif_set_promiscuity()
9211 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; in netif_set_allmulti()
9216 allmulti = dev->allmulti + inc; in netif_set_allmulti()
9224 return -EOVERFLOW; in netif_set_allmulti()
9230 WRITE_ONCE(dev->allmulti, allmulti); in netif_set_allmulti()
9232 WRITE_ONCE(dev->flags, flags); in netif_set_allmulti()
9234 dev->flags & IFF_ALLMULTI ? "entered" : "left"); in netif_set_allmulti()
9239 dev->gflags ^ old_gflags, 0, NULL); in netif_set_allmulti()
9252 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_rx_mode()
9255 if (!(dev->flags&IFF_UP)) in __dev_set_rx_mode()
9261 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { in __dev_set_rx_mode()
9265 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { in __dev_set_rx_mode()
9267 dev->uc_promisc = true; in __dev_set_rx_mode()
9268 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { in __dev_set_rx_mode()
9269 __dev_set_promiscuity(dev, -1, false); in __dev_set_rx_mode()
9270 dev->uc_promisc = false; in __dev_set_rx_mode()
9274 if (ops->ndo_set_rx_mode) in __dev_set_rx_mode()
9275 ops->ndo_set_rx_mode(dev); in __dev_set_rx_mode()
9286 * dev_get_flags - get flags reported to userspace
9295 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | in dev_get_flags()
9300 (READ_ONCE(dev->gflags) & (IFF_PROMISC | in dev_get_flags()
9319 unsigned int old_flags = dev->flags; in __dev_change_flags()
9328 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | in __dev_change_flags()
9331 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | in __dev_change_flags()
9357 if ((flags ^ dev->gflags) & IFF_PROMISC) { in __dev_change_flags()
9358 int inc = (flags & IFF_PROMISC) ? 1 : -1; in __dev_change_flags()
9359 old_flags = dev->flags; in __dev_change_flags()
9361 dev->gflags ^= IFF_PROMISC; in __dev_change_flags()
9364 if (dev->flags != old_flags) in __dev_change_flags()
9372 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { in __dev_change_flags()
9373 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; in __dev_change_flags()
9375 dev->gflags ^= IFF_ALLMULTI; in __dev_change_flags()
9386 unsigned int changes = dev->flags ^ old_flags; in __dev_notify_flags()
9392 if (dev->flags & IFF_UP) in __dev_notify_flags()
9398 if (dev->flags & IFF_UP && in __dev_notify_flags()
9415 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; in netif_change_flags()
9421 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); in netif_change_flags()
9428 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_mtu()
9430 if (ops->ndo_change_mtu) in __dev_set_mtu()
9431 return ops->ndo_change_mtu(dev, new_mtu); in __dev_set_mtu()
9433 /* Pairs with all the lockless reads of dev->mtu in the stack */ in __dev_set_mtu()
9434 WRITE_ONCE(dev->mtu, new_mtu); in __dev_set_mtu()
9443 if (new_mtu < 0 || new_mtu < dev->min_mtu) { in dev_validate_mtu()
9445 return -EINVAL; in dev_validate_mtu()
9448 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { in dev_validate_mtu()
9450 return -EINVAL; in dev_validate_mtu()
9456 * netif_set_mtu_ext - Change maximum transfer unit
9468 if (new_mtu == dev->mtu) in netif_set_mtu_ext()
9476 return -ENODEV; in netif_set_mtu_ext()
9483 orig_mtu = dev->mtu; in netif_set_mtu_ext()
9510 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); in netif_set_mtu()
9517 unsigned int orig_len = dev->tx_queue_len; in netif_change_tx_queue_len()
9521 return -ERANGE; in netif_change_tx_queue_len()
9524 WRITE_ONCE(dev->tx_queue_len, new_len); in netif_change_tx_queue_len()
9538 WRITE_ONCE(dev->tx_queue_len, orig_len); in netif_change_tx_queue_len()
9544 dev->group = new_group; in netif_set_group()
9548 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9571 const struct net_device_ops *ops = dev->netdev_ops; in netif_set_mac_address()
9574 if (!ops->ndo_set_mac_address) in netif_set_mac_address()
9575 return -EOPNOTSUPP; in netif_set_mac_address()
9576 if (sa->sa_family != dev->type) in netif_set_mac_address()
9577 return -EINVAL; in netif_set_mac_address()
9579 return -ENODEV; in netif_set_mac_address()
9580 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack); in netif_set_mac_address()
9583 if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) { in netif_set_mac_address()
9584 err = ops->ndo_set_mac_address(dev, sa); in netif_set_mac_address()
9588 dev->addr_assign_type = NET_ADDR_SET; in netif_set_mac_address()
9590 add_device_randomness(dev->dev_addr, dev->addr_len); in netif_set_mac_address()
9598 size_t size = sizeof(sa->sa_data_min); in dev_get_mac_address()
9607 ret = -ENODEV; in dev_get_mac_address()
9610 if (!dev->addr_len) in dev_get_mac_address()
9611 memset(sa->sa_data, 0, size); in dev_get_mac_address()
9613 memcpy(sa->sa_data, dev->dev_addr, in dev_get_mac_address()
9614 min_t(size_t, size, dev->addr_len)); in dev_get_mac_address()
9615 sa->sa_family = dev->type; in dev_get_mac_address()
9626 const struct net_device_ops *ops = dev->netdev_ops; in netif_change_carrier()
9628 if (!ops->ndo_change_carrier) in netif_change_carrier()
9629 return -EOPNOTSUPP; in netif_change_carrier()
9631 return -ENODEV; in netif_change_carrier()
9632 return ops->ndo_change_carrier(dev, new_carrier); in netif_change_carrier()
9636 * dev_get_phys_port_id - Get device physical port ID
9645 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_id()
9647 if (!ops->ndo_get_phys_port_id) in dev_get_phys_port_id()
9648 return -EOPNOTSUPP; in dev_get_phys_port_id()
9649 return ops->ndo_get_phys_port_id(dev, ppid); in dev_get_phys_port_id()
9653 * dev_get_phys_port_name - Get device physical port name
9663 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_name()
9666 if (ops->ndo_get_phys_port_name) { in dev_get_phys_port_name()
9667 err = ops->ndo_get_phys_port_name(dev, name, len); in dev_get_phys_port_name()
9668 if (err != -EOPNOTSUPP) in dev_get_phys_port_name()
9675 * dev_get_port_parent_id - Get the device's port parent identifier
9686 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_port_parent_id()
9692 if (ops->ndo_get_port_parent_id) { in dev_get_port_parent_id()
9693 err = ops->ndo_get_port_parent_id(dev, ppid); in dev_get_port_parent_id()
9694 if (err != -EOPNOTSUPP) in dev_get_port_parent_id()
9699 if (!recurse || err != -EOPNOTSUPP) in dev_get_port_parent_id()
9709 return -EOPNOTSUPP; in dev_get_port_parent_id()
9717 * netdev_port_same_parent_id - Indicate if two network devices have
9737 if (!dev->change_proto_down) in netif_change_proto_down()
9738 return -EOPNOTSUPP; in netif_change_proto_down()
9740 return -ENODEV; in netif_change_proto_down()
9745 WRITE_ONCE(dev->proto_down, proto_down); in netif_change_proto_down()
9750 * netdev_change_proto_down_reason_locked - proto down reason
9765 proto_down_reason = dev->proto_down_reason; in netdev_change_proto_down_reason_locked()
9773 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); in netdev_change_proto_down_reason_locked()
9790 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; in dev_xdp_mode()
9800 return dev->netdev_ops->ndo_bpf; in dev_xdp_bpf_op()
9809 return dev->xdp_state[mode].link; in dev_xdp_link()
9818 return link->link.prog; in dev_xdp_prog()
9819 return dev->xdp_state[mode].prog; in dev_xdp_prog()
9828 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) in dev_xdp_prog_count()
9840 if (dev->xdp_state[i].prog && in dev_xdp_sb_prog_count()
9841 !dev->xdp_state[i].prog->aux->xdp_has_frags) in dev_xdp_sb_prog_count()
9848 if (!dev->netdev_ops->ndo_bpf) in netif_xdp_propagate()
9849 return -EOPNOTSUPP; in netif_xdp_propagate()
9851 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in netif_xdp_propagate()
9852 bpf->command == XDP_SETUP_PROG && in netif_xdp_propagate()
9853 bpf->prog && !bpf->prog->aux->xdp_has_frags) { in netif_xdp_propagate()
9854 NL_SET_ERR_MSG(bpf->extack, in netif_xdp_propagate()
9855 "unable to propagate XDP to device using tcp-data-split"); in netif_xdp_propagate()
9856 return -EBUSY; in netif_xdp_propagate()
9860 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); in netif_xdp_propagate()
9861 return -EBUSY; in netif_xdp_propagate()
9864 return dev->netdev_ops->ndo_bpf(dev, bpf); in netif_xdp_propagate()
9871 return prog ? prog->aux->id : 0; in dev_xdp_prog_id()
9877 dev->xdp_state[mode].link = link; in dev_xdp_set_link()
9878 dev->xdp_state[mode].prog = NULL; in dev_xdp_set_link()
9884 dev->xdp_state[mode].link = NULL; in dev_xdp_set_prog()
9885 dev->xdp_state[mode].prog = prog; in dev_xdp_set_prog()
9897 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in dev_xdp_install()
9898 prog && !prog->aux->xdp_has_frags) { in dev_xdp_install()
9899 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); in dev_xdp_install()
9900 return -EBUSY; in dev_xdp_install()
9905 return -EBUSY; in dev_xdp_install()
9955 /* auto-detach link from net device */ in dev_xdp_uninstall()
9958 link->dev = NULL; in dev_xdp_uninstall()
9982 return -EINVAL; in dev_xdp_attach()
9986 return -EINVAL; in dev_xdp_attach()
9991 return -EINVAL; in dev_xdp_attach()
9997 return -EINVAL; in dev_xdp_attach()
10002 return -EINVAL; in dev_xdp_attach()
10009 return -EBUSY; in dev_xdp_attach()
10016 return -EEXIST; in dev_xdp_attach()
10024 return -EBUSY; in dev_xdp_attach()
10028 return -EEXIST; in dev_xdp_attach()
10033 new_prog = link->link.prog; in dev_xdp_attach()
10042 return -EBUSY; in dev_xdp_attach()
10046 return -EEXIST; in dev_xdp_attach()
10048 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { in dev_xdp_attach()
10050 return -EINVAL; in dev_xdp_attach()
10052 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { in dev_xdp_attach()
10054 return -EINVAL; in dev_xdp_attach()
10056 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { in dev_xdp_attach()
10057 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); in dev_xdp_attach()
10058 return -EINVAL; in dev_xdp_attach()
10060 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { in dev_xdp_attach()
10062 return -EINVAL; in dev_xdp_attach()
10064 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { in dev_xdp_attach()
10066 return -EINVAL; in dev_xdp_attach()
10075 return -EOPNOTSUPP; in dev_xdp_attach()
10097 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); in dev_xdp_attach_link()
10109 mode = dev_xdp_mode(dev, link->flags); in dev_xdp_detach_link()
10111 return -EINVAL; in dev_xdp_detach_link()
10125 /* if racing with net_device's tear down, xdp_link->dev might be in bpf_xdp_link_release()
10126 * already NULL, in which case link was already auto-detached in bpf_xdp_link_release()
10128 if (xdp_link->dev) { in bpf_xdp_link_release()
10129 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_release()
10130 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); in bpf_xdp_link_release()
10131 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_release()
10132 xdp_link->dev = NULL; in bpf_xdp_link_release()
10158 if (xdp_link->dev) in bpf_xdp_link_show_fdinfo()
10159 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_show_fdinfo()
10172 if (xdp_link->dev) in bpf_xdp_link_fill_link_info()
10173 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_fill_link_info()
10176 info->xdp.ifindex = ifindex; in bpf_xdp_link_fill_link_info()
10190 /* link might have been auto-released already, so fail */ in bpf_xdp_link_update()
10191 if (!xdp_link->dev) { in bpf_xdp_link_update()
10192 err = -ENOLINK; in bpf_xdp_link_update()
10196 if (old_prog && link->prog != old_prog) { in bpf_xdp_link_update()
10197 err = -EPERM; in bpf_xdp_link_update()
10200 old_prog = link->prog; in bpf_xdp_link_update()
10201 if (old_prog->type != new_prog->type || in bpf_xdp_link_update()
10202 old_prog->expected_attach_type != new_prog->expected_attach_type) { in bpf_xdp_link_update()
10203 err = -EINVAL; in bpf_xdp_link_update()
10208 /* no-op, don't disturb drivers */ in bpf_xdp_link_update()
10213 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_update()
10214 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); in bpf_xdp_link_update()
10215 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); in bpf_xdp_link_update()
10216 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, in bpf_xdp_link_update()
10217 xdp_link->flags, new_prog); in bpf_xdp_link_update()
10218 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_update()
10222 old_prog = xchg(&link->prog, new_prog); in bpf_xdp_link_update()
10241 struct net *net = current->nsproxy->net_ns; in bpf_xdp_link_attach()
10249 dev = dev_get_by_index(net, attr->link_create.target_ifindex); in bpf_xdp_link_attach()
10252 return -EINVAL; in bpf_xdp_link_attach()
10257 err = -ENOMEM; in bpf_xdp_link_attach()
10261 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog); in bpf_xdp_link_attach()
10262 link->dev = dev; in bpf_xdp_link_attach()
10263 link->flags = attr->link_create.flags; in bpf_xdp_link_attach()
10265 err = bpf_link_prime(&link->link, &link_primer); in bpf_xdp_link_attach()
10277 link->dev = NULL; in bpf_xdp_link_attach()
10297 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
10302 * @flags: xdp-related flags
10348 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) in dev_get_min_mp_channel_count()
10349 if (dev->_rx[i].mp_params.mp_priv) in dev_get_min_mp_channel_count()
10357 * dev_index_reserve() - allocate an ifindex in a namespace
10365 * Return: a suitable unique value for a new device interface number or -errno.
10373 return -EINVAL; in dev_index_reserve()
10377 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, in dev_index_reserve()
10378 xa_limit_31b, &net->ifindex, GFP_KERNEL); in dev_index_reserve()
10380 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); in dev_index_reserve()
10390 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); in dev_index_release()
10409 list_add_tail(&dev->todo_list, &net_todo_list); in net_set_todo()
10421 if (!(upper->wanted_features & feature) in netdev_sync_upper_features()
10424 &feature, upper->name); in netdev_sync_upper_features()
10441 if (!(features & feature) && (lower->features & feature)) { in netdev_sync_lower_features()
10443 &feature, lower->name); in netdev_sync_lower_features()
10445 lower->wanted_features &= ~feature; in netdev_sync_lower_features()
10448 if (unlikely(lower->features & feature)) in netdev_sync_lower_features()
10450 &feature, lower->name); in netdev_sync_lower_features()
10473 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); in netdev_fix_features()
10511 if ((features & dev->gso_partial_features) && in netdev_fix_features()
10515 features &= ~dev->gso_partial_features; in netdev_fix_features()
10530 /* LRO/HW-GRO features cannot be combined with RX-FCS */ in netdev_fix_features()
10533 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10538 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10544 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); in netdev_fix_features()
10549 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n"); in netdev_fix_features()
10554 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n"); in netdev_fix_features()
10571 int err = -1; in __netdev_update_features()
10578 if (dev->netdev_ops->ndo_fix_features) in __netdev_update_features()
10579 features = dev->netdev_ops->ndo_fix_features(dev, features); in __netdev_update_features()
10588 if (dev->features == features) in __netdev_update_features()
10591 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", in __netdev_update_features()
10592 &dev->features, &features); in __netdev_update_features()
10594 if (dev->netdev_ops->ndo_set_features) in __netdev_update_features()
10595 err = dev->netdev_ops->ndo_set_features(dev, features); in __netdev_update_features()
10602 err, &features, &dev->features); in __netdev_update_features()
10603 /* return non-0 since some features might have changed and in __netdev_update_features()
10606 return -1; in __netdev_update_features()
10617 netdev_features_t diff = features ^ dev->features; in __netdev_update_features()
10623 * Thus we need to update dev->features in __netdev_update_features()
10628 dev->features = features; in __netdev_update_features()
10637 dev->features = features; in __netdev_update_features()
10646 dev->features = features; in __netdev_update_features()
10653 dev->features = features; in __netdev_update_features()
10660 * netdev_update_features - recalculate device features
10663 * Recalculate dev->features set and send notifications if it
10675 * netdev_change_features - recalculate device features
10678 * Recalculate dev->features set and send notifications even
10680 * netdev_update_features() if also dev->vlan_features might
10692 * netif_stacked_transfer_operstate - transfer operstate
10703 if (rootdev->operstate == IF_OPER_DORMANT) in netif_stacked_transfer_operstate()
10708 if (rootdev->operstate == IF_OPER_TESTING) in netif_stacked_transfer_operstate()
10722 unsigned int i, count = dev->num_rx_queues; in netif_alloc_rx_queues()
10731 return -ENOMEM; in netif_alloc_rx_queues()
10733 dev->_rx = rx; in netif_alloc_rx_queues()
10738 /* XDP RX-queue setup */ in netif_alloc_rx_queues()
10747 while (i--) in netif_alloc_rx_queues()
10749 kvfree(dev->_rx); in netif_alloc_rx_queues()
10750 dev->_rx = NULL; in netif_alloc_rx_queues()
10756 unsigned int i, count = dev->num_rx_queues; in netif_free_rx_queues()
10759 if (!dev->_rx) in netif_free_rx_queues()
10763 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); in netif_free_rx_queues()
10765 kvfree(dev->_rx); in netif_free_rx_queues()
10772 spin_lock_init(&queue->_xmit_lock); in netdev_init_one_queue()
10773 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); in netdev_init_one_queue()
10774 queue->xmit_lock_owner = -1; in netdev_init_one_queue()
10776 queue->dev = dev; in netdev_init_one_queue()
10778 dql_init(&queue->dql, HZ); in netdev_init_one_queue()
10784 kvfree(dev->_tx); in netif_free_tx_queues()
10789 unsigned int count = dev->num_tx_queues; in netif_alloc_netdev_queues()
10794 return -EINVAL; in netif_alloc_netdev_queues()
10798 return -ENOMEM; in netif_alloc_netdev_queues()
10800 dev->_tx = tx; in netif_alloc_netdev_queues()
10803 spin_lock_init(&dev->tx_global_lock); in netif_alloc_netdev_queues()
10812 for (i = 0; i < dev->num_tx_queues; i++) { in netif_tx_stop_all_queues()
10828 if (dev->netdev_ops->ndo_get_peer_dev && in netdev_do_alloc_pcpu_stats()
10829 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) in netdev_do_alloc_pcpu_stats()
10830 return -EOPNOTSUPP; in netdev_do_alloc_pcpu_stats()
10832 switch (dev->pcpu_stat_type) { in netdev_do_alloc_pcpu_stats()
10836 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); in netdev_do_alloc_pcpu_stats()
10839 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); in netdev_do_alloc_pcpu_stats()
10842 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); in netdev_do_alloc_pcpu_stats()
10845 return -EINVAL; in netdev_do_alloc_pcpu_stats()
10848 return v ? 0 : -ENOMEM; in netdev_do_alloc_pcpu_stats()
10853 switch (dev->pcpu_stat_type) { in netdev_do_free_pcpu_stats()
10857 free_percpu(dev->lstats); in netdev_do_free_pcpu_stats()
10860 free_percpu(dev->tstats); in netdev_do_free_pcpu_stats()
10863 free_percpu(dev->dstats); in netdev_do_free_pcpu_stats()
10870 struct phy_link_topology *topo = dev->link_topo; in netdev_free_phy_link_topology()
10873 xa_destroy(&topo->phys); in netdev_free_phy_link_topology()
10875 dev->link_topo = NULL; in netdev_free_phy_link_topology()
10880 * register_netdevice() - register a network device
10885 * Callers must hold the rtnl lock - you may want register_netdev()
10901 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); in register_netdevice()
10904 ret = ethtool_check_ops(dev->ethtool_ops); in register_netdevice()
10909 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); in register_netdevice()
10910 mutex_init(&dev->ethtool->rss_lock); in register_netdevice()
10912 spin_lock_init(&dev->addr_list_lock); in register_netdevice()
10915 ret = dev_get_valid_name(net, dev, dev->name); in register_netdevice()
10919 ret = -ENOMEM; in register_netdevice()
10920 dev->name_node = netdev_name_node_head_alloc(dev); in register_netdevice()
10921 if (!dev->name_node) in register_netdevice()
10925 if (dev->netdev_ops->ndo_init) { in register_netdevice()
10926 ret = dev->netdev_ops->ndo_init(dev); in register_netdevice()
10929 ret = -EIO; in register_netdevice()
10934 if (((dev->hw_features | dev->features) & in register_netdevice()
10936 (!dev->netdev_ops->ndo_vlan_rx_add_vid || in register_netdevice()
10937 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { in register_netdevice()
10939 ret = -EINVAL; in register_netdevice()
10947 ret = dev_index_reserve(net, dev->ifindex); in register_netdevice()
10950 dev->ifindex = ret; in register_netdevice()
10953 * software offloads (GSO and GRO). in register_netdevice()
10955 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); in register_netdevice()
10956 dev->features |= NETIF_F_SOFT_FEATURES; in register_netdevice()
10958 if (dev->udp_tunnel_nic_info) { in register_netdevice()
10959 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
10960 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
10963 dev->wanted_features = dev->features & dev->hw_features; in register_netdevice()
10965 if (!(dev->flags & IFF_LOOPBACK)) in register_netdevice()
10966 dev->hw_features |= NETIF_F_NOCACHE_COPY; in register_netdevice()
10973 if (dev->hw_features & NETIF_F_TSO) in register_netdevice()
10974 dev->hw_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10975 if (dev->vlan_features & NETIF_F_TSO) in register_netdevice()
10976 dev->vlan_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10977 if (dev->mpls_features & NETIF_F_TSO) in register_netdevice()
10978 dev->mpls_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10979 if (dev->hw_enc_features & NETIF_F_TSO) in register_netdevice()
10980 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10984 dev->vlan_features |= NETIF_F_HIGHDMA; in register_netdevice()
10988 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; in register_netdevice()
10992 dev->mpls_features |= NETIF_F_SG; in register_netdevice()
11002 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); in register_netdevice()
11017 set_bit(__LINK_STATE_PRESENT, &dev->state); in register_netdevice()
11023 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); in register_netdevice()
11026 add_device_randomness(dev->dev_addr, dev->addr_len); in register_netdevice()
11032 if (dev->addr_assign_type == NET_ADDR_PERM) in register_netdevice()
11033 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); in register_netdevice()
11042 dev->needs_free_netdev = false; in register_netdevice()
11050 if (!dev->rtnl_link_ops || in register_netdevice()
11051 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in register_netdevice()
11060 dev_index_release(net, dev->ifindex); in register_netdevice()
11064 if (dev->netdev_ops->ndo_uninit) in register_netdevice()
11065 dev->netdev_ops->ndo_uninit(dev); in register_netdevice()
11066 if (dev->priv_destructor) in register_netdevice()
11067 dev->priv_destructor(dev); in register_netdevice()
11069 netdev_name_node_free(dev->name_node); in register_netdevice()
11083 dev->reg_state = NETREG_DUMMY; in init_dummy_netdev()
11086 set_bit(__LINK_STATE_PRESENT, &dev->state); in init_dummy_netdev()
11087 set_bit(__LINK_STATE_START, &dev->state); in init_dummy_netdev()
11096 * register_netdev - register a network device
11114 return -EINTR; in register_netdev()
11130 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); in netdev_refcnt_read()
11133 return refcount_read(&dev->dev_refcnt); in netdev_refcnt_read()
11143 * netdev_wait_allrefs_any - wait until all references are gone.
11180 &dev->state)) { in netdev_wait_allrefs_any()
11213 dev->name, netdev_refcnt_read(dev)); in netdev_wait_allrefs_any()
11214 ref_tracker_dir_print(&dev->refcnt_tracker, 10); in netdev_wait_allrefs_any()
11259 list_del_init(&dev->unlink_list); in netdev_run_todo()
11260 dev->nested_level = dev->lower_level - 1; in netdev_run_todo()
11274 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { in netdev_run_todo()
11276 list_del(&dev->todo_list); in netdev_run_todo()
11281 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); in netdev_run_todo()
11289 list_del(&dev->todo_list); in netdev_run_todo()
11293 BUG_ON(!list_empty(&dev->ptype_all)); in netdev_run_todo()
11294 BUG_ON(!list_empty(&dev->ptype_specific)); in netdev_run_todo()
11295 WARN_ON(rcu_access_pointer(dev->ip_ptr)); in netdev_run_todo()
11296 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); in netdev_run_todo()
11299 if (dev->priv_destructor) in netdev_run_todo()
11300 dev->priv_destructor(dev); in netdev_run_todo()
11301 if (dev->needs_free_netdev) in netdev_run_todo()
11307 kobject_put(&dev->dev.kobj); in netdev_run_todo()
11313 /* Collate per-cpu network dstats statistics
11315 * Read per-cpu network statistics from dev->dstats and populate the related
11331 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_dstats()
11332 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_dstats()
11333 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_dstats()
11334 rx_drops = u64_stats_read(&stats->rx_drops); in dev_fetch_dstats()
11335 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_dstats()
11336 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_dstats()
11337 tx_drops = u64_stats_read(&stats->tx_drops); in dev_fetch_dstats()
11338 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_dstats()
11340 s->rx_packets += rx_packets; in dev_fetch_dstats()
11341 s->rx_bytes += rx_bytes; in dev_fetch_dstats()
11342 s->rx_dropped += rx_drops; in dev_fetch_dstats()
11343 s->tx_packets += tx_packets; in dev_fetch_dstats()
11344 s->tx_bytes += tx_bytes; in dev_fetch_dstats()
11345 s->tx_dropped += tx_drops; in dev_fetch_dstats()
11349 /* ndo_get_stats64 implementation for dtstats-based accounting.
11351 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11352 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11357 netdev_stats_to_stats64(s, &dev->stats); in dev_get_dstats64()
11358 dev_fetch_dstats(s, dev->dstats); in dev_get_dstats64()
11378 sizeof(*stats64) - n * sizeof(u64)); in netdev_stats_to_stats64()
11390 if (p && cmpxchg(&dev->core_stats, NULL, p)) in netdev_core_stats_alloc()
11394 return READ_ONCE(dev->core_stats); in netdev_core_stats_alloc()
11400 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); in netdev_core_stats_inc()
11415 * dev_get_stats - get network device statistics
11421 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11427 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_stats()
11444 if (ops->ndo_get_stats64) { in dev_get_stats()
11446 ops->ndo_get_stats64(dev, storage); in dev_get_stats()
11447 } else if (ops->ndo_get_stats) { in dev_get_stats()
11448 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); in dev_get_stats()
11449 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { in dev_get_stats()
11451 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { in dev_get_stats()
11454 netdev_stats_to_stats64(storage, &dev->stats); in dev_get_stats()
11458 p = READ_ONCE(dev->core_stats); in dev_get_stats()
11465 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); in dev_get_stats()
11466 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); in dev_get_stats()
11467 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); in dev_get_stats()
11468 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); in dev_get_stats()
11476 * dev_fetch_sw_netstats - get per-cpu network device statistics
11478 * @netstats: per-cpu network stats to read from
11480 * Read per-cpu network statistics and populate the related fields in @s.
11494 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_sw_netstats()
11495 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_sw_netstats()
11496 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_sw_netstats()
11497 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_sw_netstats()
11498 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_sw_netstats()
11499 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_sw_netstats()
11501 s->rx_packets += rx_packets; in dev_fetch_sw_netstats()
11502 s->rx_bytes += rx_bytes; in dev_fetch_sw_netstats()
11503 s->tx_packets += tx_packets; in dev_fetch_sw_netstats()
11504 s->tx_bytes += tx_bytes; in dev_fetch_sw_netstats()
11510 * dev_get_tstats64 - ndo_get_stats64 implementation
11514 * Populate @s from dev->stats and dev->tstats. Can be used as
11519 netdev_stats_to_stats64(s, &dev->stats); in dev_get_tstats64()
11520 dev_fetch_sw_netstats(s, dev->tstats); in dev_get_tstats64()
11535 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); in dev_ingress_queue_create()
11536 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); in dev_ingress_queue_create()
11537 rcu_assign_pointer(dev->ingress_queue, queue); in dev_ingress_queue_create()
11547 if (dev->ethtool_ops == &default_ethtool_ops) in netdev_set_default_ethtool_ops()
11548 dev->ethtool_ops = ops; in netdev_set_default_ethtool_ops()
11553 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11561 WARN_ON(dev->reg_state == NETREG_REGISTERED); in netdev_sw_irq_coalesce_default_on()
11571 * alloc_netdev_mqs - allocate network device
11592 BUG_ON(strlen(name) >= sizeof(dev->name)); in alloc_netdev_mqs()
11611 dev->priv_len = sizeof_priv; in alloc_netdev_mqs()
11613 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name); in alloc_netdev_mqs()
11615 dev->pcpu_refcnt = alloc_percpu(int); in alloc_netdev_mqs()
11616 if (!dev->pcpu_refcnt) in alloc_netdev_mqs()
11620 refcount_set(&dev->dev_refcnt, 1); in alloc_netdev_mqs()
11631 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11632 dev->xdp_zc_max_segs = 1; in alloc_netdev_mqs()
11633 dev->gso_max_segs = GSO_MAX_SEGS; in alloc_netdev_mqs()
11634 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11635 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11636 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11637 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11638 dev->tso_max_segs = TSO_MAX_SEGS; in alloc_netdev_mqs()
11639 dev->upper_level = 1; in alloc_netdev_mqs()
11640 dev->lower_level = 1; in alloc_netdev_mqs()
11642 dev->nested_level = 0; in alloc_netdev_mqs()
11643 INIT_LIST_HEAD(&dev->unlink_list); in alloc_netdev_mqs()
11646 INIT_LIST_HEAD(&dev->napi_list); in alloc_netdev_mqs()
11647 INIT_LIST_HEAD(&dev->unreg_list); in alloc_netdev_mqs()
11648 INIT_LIST_HEAD(&dev->close_list); in alloc_netdev_mqs()
11649 INIT_LIST_HEAD(&dev->link_watch_list); in alloc_netdev_mqs()
11650 INIT_LIST_HEAD(&dev->adj_list.upper); in alloc_netdev_mqs()
11651 INIT_LIST_HEAD(&dev->adj_list.lower); in alloc_netdev_mqs()
11652 INIT_LIST_HEAD(&dev->ptype_all); in alloc_netdev_mqs()
11653 INIT_LIST_HEAD(&dev->ptype_specific); in alloc_netdev_mqs()
11654 INIT_LIST_HEAD(&dev->net_notifier_list); in alloc_netdev_mqs()
11656 hash_init(dev->qdisc_hash); in alloc_netdev_mqs()
11659 mutex_init(&dev->lock); in alloc_netdev_mqs()
11661 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; in alloc_netdev_mqs()
11664 if (!dev->tx_queue_len) { in alloc_netdev_mqs()
11665 dev->priv_flags |= IFF_NO_QUEUE; in alloc_netdev_mqs()
11666 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; in alloc_netdev_mqs()
11669 dev->num_tx_queues = txqs; in alloc_netdev_mqs()
11670 dev->real_num_tx_queues = txqs; in alloc_netdev_mqs()
11674 dev->num_rx_queues = rxqs; in alloc_netdev_mqs()
11675 dev->real_num_rx_queues = rxqs; in alloc_netdev_mqs()
11678 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11679 if (!dev->ethtool) in alloc_netdev_mqs()
11682 dev->cfg = kzalloc(sizeof(*dev->cfg), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11683 if (!dev->cfg) in alloc_netdev_mqs()
11685 dev->cfg_pending = dev->cfg; in alloc_netdev_mqs()
11687 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); in alloc_netdev_mqs()
11688 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11689 if (!dev->napi_config) in alloc_netdev_mqs()
11692 strscpy(dev->name, name); in alloc_netdev_mqs()
11693 dev->name_assign_type = name_assign_type; in alloc_netdev_mqs()
11694 dev->group = INIT_NETDEV_GROUP; in alloc_netdev_mqs()
11695 if (!dev->ethtool_ops) in alloc_netdev_mqs()
11696 dev->ethtool_ops = &default_ethtool_ops; in alloc_netdev_mqs()
11708 free_percpu(dev->pcpu_refcnt); in alloc_netdev_mqs()
11718 if (!list_empty(&dev->napi_list)) { in netdev_napi_exit()
11722 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) in netdev_napi_exit()
11729 kvfree(dev->napi_config); in netdev_napi_exit()
11733 * free_netdev - free network device
11749 if (dev->reg_state == NETREG_UNREGISTERING) { in free_netdev()
11751 dev->needs_free_netdev = true; in free_netdev()
11755 WARN_ON(dev->cfg != dev->cfg_pending); in free_netdev()
11756 kfree(dev->cfg); in free_netdev()
11757 kfree(dev->ethtool); in free_netdev()
11761 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); in free_netdev()
11770 ref_tracker_dir_exit(&dev->refcnt_tracker); in free_netdev()
11772 free_percpu(dev->pcpu_refcnt); in free_netdev()
11773 dev->pcpu_refcnt = NULL; in free_netdev()
11775 free_percpu(dev->core_stats); in free_netdev()
11776 dev->core_stats = NULL; in free_netdev()
11777 free_percpu(dev->xdp_bulkq); in free_netdev()
11778 dev->xdp_bulkq = NULL; in free_netdev()
11782 mutex_destroy(&dev->lock); in free_netdev()
11785 if (dev->reg_state == NETREG_UNINITIALIZED || in free_netdev()
11786 dev->reg_state == NETREG_DUMMY) { in free_netdev()
11791 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); in free_netdev()
11792 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); in free_netdev()
11795 put_device(&dev->dev); in free_netdev()
11800 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
11813 * synchronize_net - Synchronize with packet receive processing
11833 mutex_lock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11834 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { in netdev_rss_contexts_free()
11839 rxfh.hfunc = ctx->hfunc; in netdev_rss_contexts_free()
11840 rxfh.input_xfrm = ctx->input_xfrm; in netdev_rss_contexts_free()
11844 xa_erase(&dev->ethtool->rss_ctx, context); in netdev_rss_contexts_free()
11845 if (dev->ethtool_ops->create_rxfh_context) in netdev_rss_contexts_free()
11846 dev->ethtool_ops->remove_rxfh_context(dev, ctx, in netdev_rss_contexts_free()
11849 dev->ethtool_ops->set_rxfh(dev, &rxfh, NULL); in netdev_rss_contexts_free()
11852 xa_destroy(&dev->ethtool->rss_ctx); in netdev_rss_contexts_free()
11853 mutex_unlock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11857 * unregister_netdevice_queue - remove device from the kernel
11874 list_move_tail(&dev->unreg_list, head); in unregister_netdevice_queue()
11878 list_add(&dev->unreg_list, &single); in unregister_netdevice_queue()
11888 for (i = 0; i < dev->real_num_rx_queues; i++) { in dev_memory_provider_uninstall()
11889 struct netdev_rx_queue *rxq = &dev->_rx[i]; in dev_memory_provider_uninstall()
11890 struct pp_memory_provider_params *p = &rxq->mp_params; in dev_memory_provider_uninstall()
11892 if (p->mp_ops && p->mp_ops->uninstall) in dev_memory_provider_uninstall()
11893 p->mp_ops->uninstall(rxq->mp_params.mp_priv, rxq); in dev_memory_provider_uninstall()
11915 if (dev->reg_state == NETREG_UNINITIALIZED) { in unregister_netdevice_many_notify()
11917 dev->name, dev); in unregister_netdevice_many_notify()
11920 list_del(&dev->unreg_list); in unregister_netdevice_many_notify()
11923 dev->dismantle = true; in unregister_netdevice_many_notify()
11924 BUG_ON(dev->reg_state != NETREG_REGISTERED); in unregister_netdevice_many_notify()
11930 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
11940 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
11948 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); in unregister_netdevice_many_notify()
11974 if (!dev->rtnl_link_ops || in unregister_netdevice_many_notify()
11975 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in unregister_netdevice_many_notify()
11987 netdev_name_node_free(dev->name_node); in unregister_netdevice_many_notify()
11993 if (dev->netdev_ops->ndo_uninit) in unregister_netdevice_many_notify()
11994 dev->netdev_ops->ndo_uninit(dev); in unregister_netdevice_many_notify()
11996 mutex_destroy(&dev->ethtool->rss_lock); in unregister_netdevice_many_notify()
12018 netdev_put(dev, &dev->dev_registered_tracker); in unregister_netdevice_many_notify()
12028 * unregister_netdevice_many - unregister many devices
12041 * unregister_netdev - remove device from the kernel
12071 err = -EINVAL; in __dev_change_net_namespace()
12072 if (dev->netns_immutable) { in __dev_change_net_namespace()
12078 if (dev->reg_state != NETREG_REGISTERED) { in __dev_change_net_namespace()
12091 err = -EEXIST; in __dev_change_net_namespace()
12092 if (netdev_name_in_use(net, dev->name)) { in __dev_change_net_namespace()
12108 err = -EEXIST; in __dev_change_net_namespace()
12110 if (netdev_name_in_use(net, name_node->name)) { in __dev_change_net_namespace()
12113 name_node->name); in __dev_change_net_namespace()
12129 err = dev_index_reserve(net, dev->ifindex); in __dev_change_net_namespace()
12130 if (err == -EBUSY) in __dev_change_net_namespace()
12161 * Note that dev->reg_state stays at NETREG_REGISTERED. in __dev_change_net_namespace()
12179 /* Send a netdev-removed uevent to the old namespace */ in __dev_change_net_namespace()
12180 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); in __dev_change_net_namespace()
12183 /* Move per-net netdevice notifiers that are following the netdevice */ in __dev_change_net_namespace()
12188 dev->ifindex = new_ifindex; in __dev_change_net_namespace()
12193 strscpy(dev->name, new_name, IFNAMSIZ); in __dev_change_net_namespace()
12198 dev_set_uevent_suppress(&dev->dev, 1); in __dev_change_net_namespace()
12199 err = device_rename(&dev->dev, dev->name); in __dev_change_net_namespace()
12200 dev_set_uevent_suppress(&dev->dev, 0); in __dev_change_net_namespace()
12203 /* Send a netdev-add uevent to the new namespace */ in __dev_change_net_namespace()
12204 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); in __dev_change_net_namespace()
12245 list_skb = &sd->completion_queue; in dev_cpu_dead()
12247 list_skb = &(*list_skb)->next; in dev_cpu_dead()
12249 *list_skb = oldsd->completion_queue; in dev_cpu_dead()
12250 oldsd->completion_queue = NULL; in dev_cpu_dead()
12253 if (oldsd->output_queue) { in dev_cpu_dead()
12254 *sd->output_queue_tailp = oldsd->output_queue; in dev_cpu_dead()
12255 sd->output_queue_tailp = oldsd->output_queue_tailp; in dev_cpu_dead()
12256 oldsd->output_queue = NULL; in dev_cpu_dead()
12257 oldsd->output_queue_tailp = &oldsd->output_queue; in dev_cpu_dead()
12263 while (!list_empty(&oldsd->poll_list)) { in dev_cpu_dead()
12264 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead()
12268 list_del_init(&napi->poll_list); in dev_cpu_dead()
12269 if (napi->poll == process_backlog) in dev_cpu_dead()
12270 napi->state &= NAPIF_STATE_THREADED; in dev_cpu_dead()
12280 remsd = oldsd->rps_ipi_list; in dev_cpu_dead()
12281 oldsd->rps_ipi_list = NULL; in dev_cpu_dead()
12288 while ((skb = __skb_dequeue(&oldsd->process_queue))) { in dev_cpu_dead()
12292 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { in dev_cpu_dead()
12301 * netdev_increment_features - increment feature set by one
12320 /* If one device supports hw checksumming, set for all. */ in netdev_increment_features()
12347 INIT_LIST_HEAD(&net->dev_base_head); in netdev_init()
12349 net->dev_name_head = netdev_create_hash(); in netdev_init()
12350 if (net->dev_name_head == NULL) in netdev_init()
12353 net->dev_index_head = netdev_create_hash(); in netdev_init()
12354 if (net->dev_index_head == NULL) in netdev_init()
12357 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); in netdev_init()
12359 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); in netdev_init()
12364 kfree(net->dev_name_head); in netdev_init()
12366 return -ENOMEM; in netdev_init()
12370 * netdev_drivername - network driver for the device
12381 parent = dev->dev.parent; in netdev_drivername()
12385 driver = parent->driver; in netdev_drivername()
12386 if (driver && driver->name) in netdev_drivername()
12387 return driver->name; in netdev_drivername()
12394 if (dev && dev->dev.parent) { in __netdev_printk()
12395 dev_printk_emit(level[1] - '0', in __netdev_printk()
12396 dev->dev.parent, in __netdev_printk()
12398 dev_driver_string(dev->dev.parent), in __netdev_printk()
12399 dev_name(dev->dev.parent), in __netdev_printk()
12454 kfree(net->dev_name_head); in netdev_exit()
12455 kfree(net->dev_index_head); in netdev_exit()
12456 xa_destroy(&net->dev_by_index); in netdev_exit()
12458 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); in netdev_exit()
12480 if (dev->netns_immutable) in default_device_exit_net()
12484 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) in default_device_exit_net()
12488 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); in default_device_exit_net()
12493 if (netdev_name_in_use(&init_net, name_node->name)) in default_device_exit_net()
12499 __func__, dev->name, err); in default_device_exit_net()
12524 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) in default_device_exit_batch()
12525 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); in default_device_exit_batch()
12540 /* TX read-mostly hotpath */ in net_dev_struct_check()
12565 /* TXRX read-mostly hotpath */ in net_dev_struct_check()
12574 /* RX read-mostly hotpath */ in net_dev_struct_check()
12616 return -ENOMEM; in net_page_pool_create()
12632 struct napi_struct *napi = &sd->backlog; in backlog_napi_should_run()
12634 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in backlog_napi_should_run()
12641 napi_threaded_poll_loop(&sd->backlog); in run_backlog_napi()
12647 struct napi_struct *napi = &sd->backlog; in backlog_napi_setup()
12649 napi->thread = this_cpu_read(backlog_napi); in backlog_napi_setup()
12650 set_bit(NAPI_STATE_THREADED, &napi->state); in backlog_napi_setup()
12667 int i, rc = -ENOMEM; in net_dev_init()
12696 skb_queue_head_init(&sd->input_pkt_queue); in net_dev_init()
12697 skb_queue_head_init(&sd->process_queue); in net_dev_init()
12699 skb_queue_head_init(&sd->xfrm_backlog); in net_dev_init()
12701 INIT_LIST_HEAD(&sd->poll_list); in net_dev_init()
12702 sd->output_queue_tailp = &sd->output_queue; in net_dev_init()
12704 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); in net_dev_init()
12705 sd->cpu = i; in net_dev_init()
12707 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); in net_dev_init()
12708 spin_lock_init(&sd->defer_lock); in net_dev_init()
12710 gro_init(&sd->backlog.gro); in net_dev_init()
12711 sd->backlog.poll = process_backlog; in net_dev_init()
12712 sd->backlog.weight = weight_p; in net_dev_init()
12713 INIT_LIST_HEAD(&sd->backlog.poll_list); in net_dev_init()