1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * net busy poll support 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * Author: Eliezer Tamir 7 * 8 * Contact Information: 9 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 10 */ 11 12 #ifndef _LINUX_NET_BUSY_POLL_H 13 #define _LINUX_NET_BUSY_POLL_H 14 15 #include <linux/netdevice.h> 16 #include <linux/sched/clock.h> 17 #include <linux/sched/signal.h> 18 #include <net/ip.h> 19 #include <net/xdp.h> 20 21 /* 0 - Reserved to indicate value not set 22 * 1..NR_CPUS - Reserved for sender_cpu 23 * NR_CPUS+1..~0 - Region available for NAPI IDs 24 */ 25 #define MIN_NAPI_ID ((unsigned int)(NR_CPUS + 1)) 26 27 static inline bool napi_id_valid(unsigned int napi_id) 28 { 29 return napi_id >= MIN_NAPI_ID; 30 } 31 32 #define BUSY_POLL_BUDGET 8 33 34 #ifdef CONFIG_NET_RX_BUSY_POLL 35 36 struct napi_struct; 37 extern unsigned int sysctl_net_busy_read __read_mostly; 38 extern unsigned int sysctl_net_busy_poll __read_mostly; 39 40 static inline bool net_busy_loop_on(void) 41 { 42 return READ_ONCE(sysctl_net_busy_poll); 43 } 44 45 static inline bool sk_can_busy_loop(const struct sock *sk) 46 { 47 return READ_ONCE(sk->sk_ll_usec) && !signal_pending(current); 48 } 49 50 bool sk_busy_loop_end(void *p, unsigned long start_time); 51 52 void napi_busy_loop(unsigned int napi_id, 53 bool (*loop_end)(void *, unsigned long), 54 void *loop_end_arg, bool prefer_busy_poll, u16 budget); 55 56 void napi_busy_loop_rcu(unsigned int napi_id, 57 bool (*loop_end)(void *, unsigned long), 58 void *loop_end_arg, bool prefer_busy_poll, u16 budget); 59 60 void napi_suspend_irqs(unsigned int napi_id); 61 void napi_resume_irqs(unsigned int napi_id); 62 63 #else /* CONFIG_NET_RX_BUSY_POLL */ 64 static inline unsigned long net_busy_loop_on(void) 65 { 66 return 0; 67 } 68 69 static inline bool sk_can_busy_loop(struct sock *sk) 70 { 71 return false; 72 } 73 74 #endif /* CONFIG_NET_RX_BUSY_POLL */ 75 76 static inline unsigned long busy_loop_current_time(void) 77 { 78 #ifdef CONFIG_NET_RX_BUSY_POLL 79 return (unsigned long)(ktime_get_ns() >> 10); 80 #else 81 return 0; 82 #endif 83 } 84 85 /* in poll/select we use the global sysctl_net_ll_poll value */ 86 static inline bool busy_loop_timeout(unsigned long start_time) 87 { 88 #ifdef CONFIG_NET_RX_BUSY_POLL 89 unsigned long bp_usec = READ_ONCE(sysctl_net_busy_poll); 90 91 if (bp_usec) { 92 unsigned long end_time = start_time + bp_usec; 93 unsigned long now = busy_loop_current_time(); 94 95 return time_after(now, end_time); 96 } 97 #endif 98 return true; 99 } 100 101 static inline bool sk_busy_loop_timeout(struct sock *sk, 102 unsigned long start_time) 103 { 104 #ifdef CONFIG_NET_RX_BUSY_POLL 105 unsigned long bp_usec = READ_ONCE(sk->sk_ll_usec); 106 107 if (bp_usec) { 108 unsigned long end_time = start_time + bp_usec; 109 unsigned long now = busy_loop_current_time(); 110 111 return time_after(now, end_time); 112 } 113 #endif 114 return true; 115 } 116 117 static inline void sk_busy_loop(struct sock *sk, int nonblock) 118 { 119 #ifdef CONFIG_NET_RX_BUSY_POLL 120 unsigned int napi_id = READ_ONCE(sk->sk_napi_id); 121 122 if (napi_id_valid(napi_id)) 123 napi_busy_loop(napi_id, nonblock ? NULL : sk_busy_loop_end, sk, 124 READ_ONCE(sk->sk_prefer_busy_poll), 125 READ_ONCE(sk->sk_busy_poll_budget) ?: BUSY_POLL_BUDGET); 126 #endif 127 } 128 129 /* used in the NIC receive handler to mark the skb */ 130 static inline void skb_mark_napi_id(struct sk_buff *skb, 131 struct napi_struct *napi) 132 { 133 #ifdef CONFIG_NET_RX_BUSY_POLL 134 /* If the skb was already marked with a valid NAPI ID, avoid overwriting 135 * it. 136 */ 137 if (!napi_id_valid(skb->napi_id)) 138 skb->napi_id = napi->napi_id; 139 #endif 140 } 141 142 /* used in the protocol handler to propagate the napi_id to the socket */ 143 static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb) 144 { 145 #ifdef CONFIG_NET_RX_BUSY_POLL 146 if (unlikely(READ_ONCE(sk->sk_napi_id) != skb->napi_id)) 147 WRITE_ONCE(sk->sk_napi_id, skb->napi_id); 148 #endif 149 sk_rx_queue_update(sk, skb); 150 } 151 152 /* Variant of sk_mark_napi_id() for passive flow setup, 153 * as sk->sk_napi_id and sk->sk_rx_queue_mapping content 154 * needs to be set. 155 */ 156 static inline void sk_mark_napi_id_set(struct sock *sk, 157 const struct sk_buff *skb) 158 { 159 #ifdef CONFIG_NET_RX_BUSY_POLL 160 WRITE_ONCE(sk->sk_napi_id, skb->napi_id); 161 #endif 162 sk_rx_queue_set(sk, skb); 163 } 164 165 static inline void __sk_mark_napi_id_once(struct sock *sk, unsigned int napi_id) 166 { 167 #ifdef CONFIG_NET_RX_BUSY_POLL 168 if (!READ_ONCE(sk->sk_napi_id)) 169 WRITE_ONCE(sk->sk_napi_id, napi_id); 170 #endif 171 } 172 173 /* variant used for unconnected sockets */ 174 static inline void sk_mark_napi_id_once(struct sock *sk, 175 const struct sk_buff *skb) 176 { 177 #ifdef CONFIG_NET_RX_BUSY_POLL 178 __sk_mark_napi_id_once(sk, skb->napi_id); 179 #endif 180 } 181 182 #endif /* _LINUX_NET_BUSY_POLL_H */ 183