1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * IPV6 GSO/GRO offload support
4 * Linux INET6 implementation
5 */
6
7 #include <linux/kernel.h>
8 #include <linux/socket.h>
9 #include <linux/netdevice.h>
10 #include <linux/skbuff.h>
11 #include <linux/printk.h>
12
13 #include <net/protocol.h>
14 #include <net/ipv6.h>
15 #include <net/inet_common.h>
16 #include <net/tcp.h>
17 #include <net/udp.h>
18 #include <net/gro.h>
19 #include <net/gso.h>
20
21 #include "ip6_offload.h"
22 #include "tcpv6_offload.c"
23
ipv6_gro_pull_exthdrs(struct sk_buff * skb,int off,int proto)24 static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto)
25 {
26 const struct net_offload *ops = NULL;
27 struct ipv6_opt_hdr *opth;
28
29 for (;;) {
30 int len;
31
32 ops = rcu_dereference(inet6_offloads[proto]);
33
34 if (unlikely(!ops))
35 break;
36
37 if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
38 break;
39
40 opth = skb_gro_header(skb, off + sizeof(*opth), off);
41 if (unlikely(!opth))
42 break;
43
44 len = ipv6_optlen(opth);
45
46 opth = skb_gro_header(skb, off + len, off);
47 if (unlikely(!opth))
48 break;
49 proto = opth->nexthdr;
50
51 off += len;
52 }
53
54 skb_gro_pull(skb, off - skb_gro_receive_network_offset(skb));
55 return proto;
56 }
57
ipv6_gso_pull_exthdrs(struct sk_buff * skb,int proto)58 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
59 {
60 const struct net_offload *ops = NULL;
61
62 for (;;) {
63 struct ipv6_opt_hdr *opth;
64 int len;
65
66 ops = rcu_dereference(inet6_offloads[proto]);
67
68 if (unlikely(!ops))
69 break;
70
71 if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
72 break;
73
74 if (unlikely(!pskb_may_pull(skb, 8)))
75 break;
76
77 opth = (void *)skb->data;
78 len = ipv6_optlen(opth);
79
80 if (unlikely(!pskb_may_pull(skb, len)))
81 break;
82
83 opth = (void *)skb->data;
84 proto = opth->nexthdr;
85 __skb_pull(skb, len);
86 }
87
88 return proto;
89 }
90
ipv6_gso_segment(struct sk_buff * skb,netdev_features_t features)91 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
92 netdev_features_t features)
93 {
94 struct sk_buff *segs = ERR_PTR(-EINVAL);
95 struct ipv6hdr *ipv6h;
96 const struct net_offload *ops;
97 int proto;
98 struct frag_hdr *fptr;
99 unsigned int payload_len;
100 u8 *prevhdr;
101 int offset = 0;
102 bool encap, udpfrag;
103 int nhoff;
104 bool gso_partial;
105
106 skb_reset_network_header(skb);
107 nhoff = skb_network_header(skb) - skb_mac_header(skb);
108 if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
109 goto out;
110
111 encap = SKB_GSO_CB(skb)->encap_level > 0;
112 if (encap)
113 features &= skb->dev->hw_enc_features;
114 SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);
115
116 ipv6h = ipv6_hdr(skb);
117 __skb_pull(skb, sizeof(*ipv6h));
118 segs = ERR_PTR(-EPROTONOSUPPORT);
119
120 proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
121
122 if (skb->encapsulation &&
123 skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))
124 udpfrag = proto == IPPROTO_UDP && encap &&
125 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
126 else
127 udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&
128 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
129
130 ops = rcu_dereference(inet6_offloads[proto]);
131 if (likely(ops && ops->callbacks.gso_segment)) {
132 if (!skb_reset_transport_header_careful(skb))
133 goto out;
134
135 segs = ops->callbacks.gso_segment(skb, features);
136 if (!segs)
137 skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
138 }
139
140 if (IS_ERR_OR_NULL(segs))
141 goto out;
142
143 gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
144
145 for (skb = segs; skb; skb = skb->next) {
146 ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
147 if (gso_partial && skb_is_gso(skb))
148 payload_len = skb_shinfo(skb)->gso_size +
149 SKB_GSO_CB(skb)->data_offset +
150 skb->head - (unsigned char *)(ipv6h + 1);
151 else
152 payload_len = skb->len - nhoff - sizeof(*ipv6h);
153 ipv6h->payload_len = htons(payload_len);
154 skb->network_header = (u8 *)ipv6h - skb->head;
155 skb_reset_mac_len(skb);
156
157 if (udpfrag) {
158 int err = ip6_find_1stfragopt(skb, &prevhdr);
159 if (err < 0) {
160 kfree_skb_list(segs);
161 return ERR_PTR(err);
162 }
163 fptr = (struct frag_hdr *)((u8 *)ipv6h + err);
164 fptr->frag_off = htons(offset);
165 if (skb->next)
166 fptr->frag_off |= htons(IP6_MF);
167 offset += (ntohs(ipv6h->payload_len) -
168 sizeof(struct frag_hdr));
169 }
170 if (encap)
171 skb_reset_inner_headers(skb);
172 }
173
174 out:
175 return segs;
176 }
177
178 /* Return the total length of all the extension hdrs, following the same
179 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
180 */
ipv6_exthdrs_len(struct ipv6hdr * iph,const struct net_offload ** opps)181 static int ipv6_exthdrs_len(struct ipv6hdr *iph,
182 const struct net_offload **opps)
183 {
184 struct ipv6_opt_hdr *opth = (void *)iph;
185 int len = 0, proto, optlen = sizeof(*iph);
186
187 proto = iph->nexthdr;
188 for (;;) {
189 *opps = rcu_dereference(inet6_offloads[proto]);
190 if (unlikely(!(*opps)))
191 break;
192 if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR))
193 break;
194
195 opth = (void *)opth + optlen;
196 optlen = ipv6_optlen(opth);
197 len += optlen;
198 proto = opth->nexthdr;
199 }
200 return len;
201 }
202
ipv6_gro_receive(struct list_head * head,struct sk_buff * skb)203 INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
204 struct sk_buff *skb)
205 {
206 const struct net_offload *ops;
207 struct sk_buff *pp = NULL;
208 struct sk_buff *p;
209 struct ipv6hdr *iph;
210 unsigned int nlen;
211 unsigned int hlen;
212 unsigned int off;
213 u16 flush = 1;
214 int proto;
215
216 off = skb_gro_offset(skb);
217 hlen = off + sizeof(*iph);
218 iph = skb_gro_header(skb, hlen, off);
219 if (unlikely(!iph))
220 goto out;
221
222 NAPI_GRO_CB(skb)->network_offsets[NAPI_GRO_CB(skb)->encap_mark] = off;
223
224 flush += ntohs(iph->payload_len) != skb->len - hlen;
225
226 proto = iph->nexthdr;
227 ops = rcu_dereference(inet6_offloads[proto]);
228 if (!ops || !ops->callbacks.gro_receive) {
229 proto = ipv6_gro_pull_exthdrs(skb, hlen, proto);
230
231 ops = rcu_dereference(inet6_offloads[proto]);
232 if (!ops || !ops->callbacks.gro_receive)
233 goto out;
234
235 iph = skb_gro_network_header(skb);
236 } else {
237 skb_gro_pull(skb, sizeof(*iph));
238 }
239
240 skb_set_transport_header(skb, skb_gro_offset(skb));
241
242 NAPI_GRO_CB(skb)->proto = proto;
243
244 flush--;
245 nlen = skb_gro_offset(skb) - off;
246
247 list_for_each_entry(p, head, list) {
248 const struct ipv6hdr *iph2;
249 __be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
250
251 if (!NAPI_GRO_CB(p)->same_flow)
252 continue;
253
254 iph2 = (struct ipv6hdr *)(p->data + off);
255 first_word = *(__be32 *)iph ^ *(__be32 *)iph2;
256
257 /* All fields must match except length and Traffic Class.
258 * XXX skbs on the gro_list have all been parsed and pulled
259 * already so we don't need to compare nlen
260 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
261 * memcmp() alone below is sufficient, right?
262 */
263 if ((first_word & htonl(0xF00FFFFF)) ||
264 !ipv6_addr_equal(&iph->saddr, &iph2->saddr) ||
265 !ipv6_addr_equal(&iph->daddr, &iph2->daddr) ||
266 iph->nexthdr != iph2->nexthdr) {
267 not_same_flow:
268 NAPI_GRO_CB(p)->same_flow = 0;
269 continue;
270 }
271 if (unlikely(nlen > sizeof(struct ipv6hdr))) {
272 if (memcmp(iph + 1, iph2 + 1,
273 nlen - sizeof(struct ipv6hdr)))
274 goto not_same_flow;
275 }
276 }
277
278 NAPI_GRO_CB(skb)->flush |= flush;
279
280 skb_gro_postpull_rcsum(skb, iph, nlen);
281
282 if (unlikely(gro_recursion_inc_test(skb))) {
283 flush = 1;
284 goto out;
285 }
286
287 if (likely(proto == IPPROTO_TCP))
288 pp = tcp6_gro_receive(head, skb);
289 #if IS_BUILTIN(CONFIG_IPV6)
290 else if (likely(proto == IPPROTO_UDP))
291 pp = udp6_gro_receive(head, skb);
292 #endif
293 else
294 pp = ops->callbacks.gro_receive(head, skb);
295 out:
296 skb_gro_flush_final(skb, pp, flush);
297
298 return pp;
299 }
300
sit_ip6ip6_gro_receive(struct list_head * head,struct sk_buff * skb)301 static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
302 struct sk_buff *skb)
303 {
304 /* Common GRO receive for SIT and IP6IP6 */
305
306 if (NAPI_GRO_CB(skb)->encap_mark) {
307 NAPI_GRO_CB(skb)->flush = 1;
308 return NULL;
309 }
310
311 NAPI_GRO_CB(skb)->encap_mark = 1;
312
313 return ipv6_gro_receive(head, skb);
314 }
315
ip4ip6_gro_receive(struct list_head * head,struct sk_buff * skb)316 static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
317 struct sk_buff *skb)
318 {
319 /* Common GRO receive for SIT and IP6IP6 */
320
321 if (NAPI_GRO_CB(skb)->encap_mark) {
322 NAPI_GRO_CB(skb)->flush = 1;
323 return NULL;
324 }
325
326 NAPI_GRO_CB(skb)->encap_mark = 1;
327
328 return inet_gro_receive(head, skb);
329 }
330
ipv6_gro_complete(struct sk_buff * skb,int nhoff)331 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
332 {
333 const struct net_offload *ops;
334 struct ipv6hdr *iph;
335 int err = -ENOSYS;
336
337 if (skb->encapsulation) {
338 skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
339 skb_set_inner_network_header(skb, nhoff);
340 }
341
342 iph = (struct ipv6hdr *)(skb->data + nhoff);
343 ipv6_set_payload_len(iph, skb->len - nhoff - sizeof(*iph));
344
345 nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
346
347 if (likely(ops == &net_hotdata.tcpv6_offload))
348 return tcp6_gro_complete(skb, nhoff);
349 #if IS_BUILTIN(CONFIG_IPV6)
350 if (ops == &net_hotdata.udpv6_offload)
351 return udp6_gro_complete(skb, nhoff);
352 #endif
353
354 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
355 goto out;
356
357 err = ops->callbacks.gro_complete(skb, nhoff);
358
359 out:
360 return err;
361 }
362
sit_gro_complete(struct sk_buff * skb,int nhoff)363 static int sit_gro_complete(struct sk_buff *skb, int nhoff)
364 {
365 skb->encapsulation = 1;
366 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
367 return ipv6_gro_complete(skb, nhoff);
368 }
369
ip6ip6_gro_complete(struct sk_buff * skb,int nhoff)370 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)
371 {
372 skb->encapsulation = 1;
373 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
374 return ipv6_gro_complete(skb, nhoff);
375 }
376
ip4ip6_gro_complete(struct sk_buff * skb,int nhoff)377 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)
378 {
379 skb->encapsulation = 1;
380 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
381 return inet_gro_complete(skb, nhoff);
382 }
383
384
sit_gso_segment(struct sk_buff * skb,netdev_features_t features)385 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
386 netdev_features_t features)
387 {
388 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
389 return ERR_PTR(-EINVAL);
390
391 return ipv6_gso_segment(skb, features);
392 }
393
ip4ip6_gso_segment(struct sk_buff * skb,netdev_features_t features)394 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,
395 netdev_features_t features)
396 {
397 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
398 return ERR_PTR(-EINVAL);
399
400 return inet_gso_segment(skb, features);
401 }
402
ip6ip6_gso_segment(struct sk_buff * skb,netdev_features_t features)403 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,
404 netdev_features_t features)
405 {
406 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
407 return ERR_PTR(-EINVAL);
408
409 return ipv6_gso_segment(skb, features);
410 }
411
412 static const struct net_offload sit_offload = {
413 .callbacks = {
414 .gso_segment = sit_gso_segment,
415 .gro_receive = sit_ip6ip6_gro_receive,
416 .gro_complete = sit_gro_complete,
417 },
418 };
419
420 static const struct net_offload ip4ip6_offload = {
421 .callbacks = {
422 .gso_segment = ip4ip6_gso_segment,
423 .gro_receive = ip4ip6_gro_receive,
424 .gro_complete = ip4ip6_gro_complete,
425 },
426 };
427
428 static const struct net_offload ip6ip6_offload = {
429 .callbacks = {
430 .gso_segment = ip6ip6_gso_segment,
431 .gro_receive = sit_ip6ip6_gro_receive,
432 .gro_complete = ip6ip6_gro_complete,
433 },
434 };
ipv6_offload_init(void)435 static int __init ipv6_offload_init(void)
436 {
437
438 if (tcpv6_offload_init() < 0)
439 pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
440 if (ipv6_exthdrs_offload_init() < 0)
441 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
442
443 net_hotdata.ipv6_packet_offload = (struct packet_offload) {
444 .type = cpu_to_be16(ETH_P_IPV6),
445 .callbacks = {
446 .gso_segment = ipv6_gso_segment,
447 .gro_receive = ipv6_gro_receive,
448 .gro_complete = ipv6_gro_complete,
449 },
450 };
451 dev_add_offload(&net_hotdata.ipv6_packet_offload);
452
453 inet_add_offload(&sit_offload, IPPROTO_IPV6);
454 inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
455 inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP);
456
457 return 0;
458 }
459
460 fs_initcall(ipv6_offload_init);
461