1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2023 Isovalent */
3 #ifndef __NET_TCX_H
4 #define __NET_TCX_H
5
6 #include <linux/bpf.h>
7 #include <linux/bpf_mprog.h>
8
9 #include <net/sch_generic.h>
10
11 struct mini_Qdisc;
12
13 struct tcx_entry {
14 struct mini_Qdisc __rcu *miniq;
15 struct bpf_mprog_bundle bundle;
16 u32 miniq_active;
17 struct rcu_head rcu;
18 };
19
20 struct tcx_link {
21 struct bpf_link link;
22 struct net_device *dev;
23 };
24
tcx_set_ingress(struct sk_buff * skb,bool ingress)25 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress)
26 {
27 #ifdef CONFIG_NET_XGRESS
28 skb->tc_at_ingress = ingress;
29 #endif
30 }
31
32 #ifdef CONFIG_NET_XGRESS
tcx_entry(struct bpf_mprog_entry * entry)33 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry)
34 {
35 struct bpf_mprog_bundle *bundle = entry->parent;
36
37 return container_of(bundle, struct tcx_entry, bundle);
38 }
39
tcx_link(const struct bpf_link * link)40 static inline struct tcx_link *tcx_link(const struct bpf_link *link)
41 {
42 return container_of(link, struct tcx_link, link);
43 }
44
45 void tcx_inc(void);
46 void tcx_dec(void);
47
tcx_entry_sync(void)48 static inline void tcx_entry_sync(void)
49 {
50 /* bpf_mprog_entry got a/b swapped, therefore ensure that
51 * there are no inflight users on the old one anymore.
52 */
53 synchronize_rcu();
54 }
55
56 static inline void
tcx_entry_update(struct net_device * dev,struct bpf_mprog_entry * entry,bool ingress)57 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry,
58 bool ingress)
59 {
60 ASSERT_RTNL();
61 if (ingress)
62 rcu_assign_pointer(dev->tcx_ingress, entry);
63 else
64 rcu_assign_pointer(dev->tcx_egress, entry);
65 }
66
67 static inline struct bpf_mprog_entry *
tcx_entry_fetch(struct net_device * dev,bool ingress)68 tcx_entry_fetch(struct net_device *dev, bool ingress)
69 {
70 ASSERT_RTNL();
71 if (ingress)
72 return rcu_dereference_rtnl(dev->tcx_ingress);
73 else
74 return rcu_dereference_rtnl(dev->tcx_egress);
75 }
76
tcx_entry_create_noprof(void)77 static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void)
78 {
79 struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL);
80
81 if (tcx) {
82 bpf_mprog_bundle_init(&tcx->bundle);
83 return &tcx->bundle.a;
84 }
85 return NULL;
86 }
87 #define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__))
88
tcx_entry_free(struct bpf_mprog_entry * entry)89 static inline void tcx_entry_free(struct bpf_mprog_entry *entry)
90 {
91 kfree_rcu(tcx_entry(entry), rcu);
92 }
93
94 static inline struct bpf_mprog_entry *
tcx_entry_fetch_or_create(struct net_device * dev,bool ingress,bool * created)95 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created)
96 {
97 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress);
98
99 *created = false;
100 if (!entry) {
101 entry = tcx_entry_create();
102 if (!entry)
103 return NULL;
104 *created = true;
105 }
106 return entry;
107 }
108
tcx_skeys_inc(bool ingress)109 static inline void tcx_skeys_inc(bool ingress)
110 {
111 tcx_inc();
112 if (ingress)
113 net_inc_ingress_queue();
114 else
115 net_inc_egress_queue();
116 }
117
tcx_skeys_dec(bool ingress)118 static inline void tcx_skeys_dec(bool ingress)
119 {
120 if (ingress)
121 net_dec_ingress_queue();
122 else
123 net_dec_egress_queue();
124 tcx_dec();
125 }
126
tcx_miniq_inc(struct bpf_mprog_entry * entry)127 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry)
128 {
129 ASSERT_RTNL();
130 tcx_entry(entry)->miniq_active++;
131 }
132
tcx_miniq_dec(struct bpf_mprog_entry * entry)133 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry)
134 {
135 ASSERT_RTNL();
136 tcx_entry(entry)->miniq_active--;
137 }
138
tcx_entry_is_active(struct bpf_mprog_entry * entry)139 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry)
140 {
141 ASSERT_RTNL();
142 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active;
143 }
144
tcx_action_code(struct sk_buff * skb,int code)145 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb,
146 int code)
147 {
148 switch (code) {
149 case TCX_PASS:
150 skb->tc_index = qdisc_skb_cb(skb)->tc_classid;
151 fallthrough;
152 case TCX_DROP:
153 case TCX_REDIRECT:
154 return code;
155 case TCX_NEXT:
156 default:
157 return TCX_NEXT;
158 }
159 }
160 #endif /* CONFIG_NET_XGRESS */
161
162 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL)
163 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog);
164 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
165 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog);
166 void tcx_uninstall(struct net_device *dev, bool ingress);
167
168 int tcx_prog_query(const union bpf_attr *attr,
169 union bpf_attr __user *uattr);
170
dev_tcx_uninstall(struct net_device * dev)171 static inline void dev_tcx_uninstall(struct net_device *dev)
172 {
173 ASSERT_RTNL();
174 tcx_uninstall(dev, true);
175 tcx_uninstall(dev, false);
176 }
177 #else
tcx_prog_attach(const union bpf_attr * attr,struct bpf_prog * prog)178 static inline int tcx_prog_attach(const union bpf_attr *attr,
179 struct bpf_prog *prog)
180 {
181 return -EINVAL;
182 }
183
tcx_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)184 static inline int tcx_link_attach(const union bpf_attr *attr,
185 struct bpf_prog *prog)
186 {
187 return -EINVAL;
188 }
189
tcx_prog_detach(const union bpf_attr * attr,struct bpf_prog * prog)190 static inline int tcx_prog_detach(const union bpf_attr *attr,
191 struct bpf_prog *prog)
192 {
193 return -EINVAL;
194 }
195
tcx_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)196 static inline int tcx_prog_query(const union bpf_attr *attr,
197 union bpf_attr __user *uattr)
198 {
199 return -EINVAL;
200 }
201
dev_tcx_uninstall(struct net_device * dev)202 static inline void dev_tcx_uninstall(struct net_device *dev)
203 {
204 }
205 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */
206 #endif /* __NET_TCX_H */
207