1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
5 * Copyright (c) 2010-2011 Juniper Networks, Inc.
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Robert N. M. Watson under
9 * contract to Juniper Networks, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. Neither the name of the project nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 *
35 * $KAME: in6_pcb.c,v 1.31 2001/05/21 05:45:10 jinmei Exp $
36 */
37
38 /*-
39 * Copyright (c) 1982, 1986, 1991, 1993
40 * The Regents of the University of California. All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 */
66
67 #include "opt_inet.h"
68 #include "opt_inet6.h"
69 #include "opt_ipsec.h"
70 #include "opt_route.h"
71 #include "opt_rss.h"
72
73 #include <sys/hash.h>
74 #include <sys/param.h>
75 #include <sys/systm.h>
76 #include <sys/malloc.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/proc.h>
80 #include <sys/protosw.h>
81 #include <sys/smr.h>
82 #include <sys/socket.h>
83 #include <sys/socketvar.h>
84 #include <sys/sockio.h>
85 #include <sys/sysctl.h>
86 #include <sys/errno.h>
87 #include <sys/time.h>
88 #include <sys/priv.h>
89 #include <sys/proc.h>
90 #include <sys/jail.h>
91
92 #include <vm/uma.h>
93
94 #include <net/if.h>
95 #include <net/if_var.h>
96 #include <net/if_llatbl.h>
97 #include <net/if_types.h>
98 #include <net/route.h>
99 #include <net/route/nhop.h>
100 #include <net/vnet.h>
101
102 #include <netinet/in.h>
103 #include <netinet/in_var.h>
104 #include <netinet/in_systm.h>
105 #include <netinet/ip6.h>
106 #include <netinet/ip_var.h>
107
108 #include <netinet6/ip6_var.h>
109 #include <netinet6/nd6.h>
110 #include <netinet/in_pcb.h>
111 #include <netinet/in_pcb_var.h>
112 #include <netinet6/in6_pcb.h>
113 #include <netinet6/in6_fib.h>
114 #include <netinet6/scope6_var.h>
115
116 int
in6_pcbsetport(struct in6_addr * laddr,struct inpcb * inp,struct ucred * cred)117 in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct ucred *cred)
118 {
119 struct socket *so = inp->inp_socket;
120 u_int16_t lport = 0;
121 int error, lookupflags = 0;
122 #ifdef INVARIANTS
123 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
124 #endif
125
126 INP_WLOCK_ASSERT(inp);
127 INP_HASH_WLOCK_ASSERT(pcbinfo);
128
129 error = prison_local_ip6(cred, laddr,
130 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0));
131 if (error)
132 return(error);
133
134 /* XXX: this is redundant when called from in6_pcbbind */
135 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
136 lookupflags = INPLOOKUP_WILDCARD;
137
138 inp->inp_flags |= INP_ANONPORT;
139
140 error = in_pcb_lport(inp, NULL, &lport, cred, lookupflags);
141 if (error != 0)
142 return (error);
143
144 inp->inp_lport = lport;
145 if (in_pcbinshash(inp) != 0) {
146 inp->in6p_laddr = in6addr_any;
147 inp->inp_lport = 0;
148 return (EAGAIN);
149 }
150
151 return (0);
152 }
153
154 /*
155 * Determine whether the inpcb can be bound to the specified address/port tuple.
156 */
157 static int
in6_pcbbind_avail(struct inpcb * inp,const struct sockaddr_in6 * sin6,int fib,int sooptions,int lookupflags,struct ucred * cred)158 in6_pcbbind_avail(struct inpcb *inp, const struct sockaddr_in6 *sin6, int fib,
159 int sooptions, int lookupflags, struct ucred *cred)
160 {
161 const struct in6_addr *laddr;
162 int reuseport, reuseport_lb;
163 u_short lport;
164
165 INP_LOCK_ASSERT(inp);
166 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
167
168 laddr = &sin6->sin6_addr;
169 lport = sin6->sin6_port;
170
171 reuseport = (sooptions & SO_REUSEPORT);
172 reuseport_lb = (sooptions & SO_REUSEPORT_LB);
173
174 if (IN6_IS_ADDR_MULTICAST(laddr)) {
175 /*
176 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
177 * allow compepte duplication of binding if
178 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
179 * and a multicast address is bound on both
180 * new and duplicated sockets.
181 */
182 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0)
183 reuseport = SO_REUSEADDR | SO_REUSEPORT;
184 /*
185 * XXX: How to deal with SO_REUSEPORT_LB here?
186 * Treat same as SO_REUSEPORT for now.
187 */
188 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0)
189 reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB;
190 } else if (!IN6_IS_ADDR_UNSPECIFIED(laddr)) {
191 struct sockaddr_in6 sin6;
192 struct epoch_tracker et;
193 struct ifaddr *ifa;
194
195 memset(&sin6, 0, sizeof(sin6));
196 sin6.sin6_family = AF_INET6;
197 sin6.sin6_len = sizeof(sin6);
198 sin6.sin6_addr = *laddr;
199
200 NET_EPOCH_ENTER(et);
201 if ((ifa = ifa_ifwithaddr((const struct sockaddr *)&sin6)) ==
202 NULL && (inp->inp_flags & INP_BINDANY) == 0) {
203 NET_EPOCH_EXIT(et);
204 return (EADDRNOTAVAIL);
205 }
206
207 /*
208 * We used to prohibit binding to an anycast address here,
209 * based on RFC3513, but that restriction was removed in
210 * RFC4291.
211 */
212 if (ifa != NULL &&
213 ((struct in6_ifaddr *)ifa)->ia6_flags &
214 (IN6_IFF_NOTREADY | IN6_IFF_DETACHED)) {
215 NET_EPOCH_EXIT(et);
216 return (EADDRNOTAVAIL);
217 }
218 NET_EPOCH_EXIT(et);
219 }
220
221 if (lport != 0) {
222 struct inpcb *t;
223
224 if (ntohs(lport) <= V_ipport_reservedhigh &&
225 ntohs(lport) >= V_ipport_reservedlow &&
226 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
227 return (EACCES);
228
229 if (!IN6_IS_ADDR_MULTICAST(laddr) &&
230 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) !=
231 0) {
232 /*
233 * If a socket owned by a different user is already
234 * bound to this port, fail. In particular, SO_REUSE*
235 * can only be used to share a port among sockets owned
236 * by the same user.
237 *
238 * However, we can share a port with a connected socket
239 * which has a unique 4-tuple.
240 */
241 t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
242 RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
243 if (t != NULL &&
244 (inp->inp_socket->so_type != SOCK_STREAM ||
245 IN6_IS_ADDR_UNSPECIFIED(&t->in6p_faddr)) &&
246 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
247 return (EADDRINUSE);
248
249 #ifdef INET
250 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
251 IN6_IS_ADDR_UNSPECIFIED(laddr)) {
252 struct sockaddr_in sin;
253
254 in6_sin6_2_sin(&sin, sin6);
255 t = in_pcblookup_local(inp->inp_pcbinfo,
256 sin.sin_addr, lport, RT_ALL_FIBS,
257 INPLOOKUP_WILDCARD, cred);
258 if (t != NULL &&
259 (inp->inp_socket->so_type != SOCK_STREAM ||
260 in_nullhost(t->inp_faddr)) &&
261 (inp->inp_cred->cr_uid !=
262 t->inp_cred->cr_uid))
263 return (EADDRINUSE);
264 }
265 #endif
266 }
267 t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
268 fib, lookupflags, cred);
269 if (t != NULL && ((reuseport | reuseport_lb) &
270 t->inp_socket->so_options) == 0)
271 return (EADDRINUSE);
272 #ifdef INET
273 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
274 IN6_IS_ADDR_UNSPECIFIED(laddr)) {
275 struct sockaddr_in sin;
276
277 in6_sin6_2_sin(&sin, sin6);
278 t = in_pcblookup_local(inp->inp_pcbinfo, sin.sin_addr,
279 lport, RT_ALL_FIBS, lookupflags, cred);
280 if (t != NULL && ((reuseport | reuseport_lb) &
281 t->inp_socket->so_options) == 0 &&
282 (!in_nullhost(t->inp_laddr) ||
283 (t->inp_vflag & INP_IPV6PROTO) != 0)) {
284 return (EADDRINUSE);
285 }
286 }
287 #endif
288 }
289 return (0);
290 }
291
292 int
in6_pcbbind(struct inpcb * inp,struct sockaddr_in6 * sin6,int flags,struct ucred * cred)293 in6_pcbbind(struct inpcb *inp, struct sockaddr_in6 *sin6, int flags,
294 struct ucred *cred)
295 {
296 struct socket *so = inp->inp_socket;
297 u_short lport = 0;
298 int error, fib, lookupflags, sooptions;
299
300 INP_WLOCK_ASSERT(inp);
301 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
302
303 if (inp->inp_lport || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
304 return (EINVAL);
305
306 lookupflags = 0;
307 sooptions = atomic_load_int(&so->so_options);
308 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0)
309 lookupflags = INPLOOKUP_WILDCARD;
310 if (sin6 == NULL) {
311 if ((error = prison_local_ip6(cred, &inp->in6p_laddr,
312 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
313 return (error);
314 } else {
315 KASSERT(sin6->sin6_family == AF_INET6,
316 ("%s: invalid address family for %p", __func__, sin6));
317 KASSERT(sin6->sin6_len == sizeof(*sin6),
318 ("%s: invalid address length for %p", __func__, sin6));
319
320 if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
321 return(error);
322
323 if ((error = prison_local_ip6(cred, &sin6->sin6_addr,
324 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
325 return (error);
326
327 fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum :
328 RT_ALL_FIBS;
329
330 /* See if this address/port combo is available. */
331 error = in6_pcbbind_avail(inp, sin6, fib, sooptions, lookupflags,
332 cred);
333 if (error != 0)
334 return (error);
335
336 lport = sin6->sin6_port;
337 inp->in6p_laddr = sin6->sin6_addr;
338 }
339 if ((flags & INPBIND_FIB) != 0)
340 inp->inp_flags |= INP_BOUNDFIB;
341 if (lport == 0) {
342 if ((error = in6_pcbsetport(&inp->in6p_laddr, inp, cred)) != 0) {
343 /* Undo an address bind that may have occurred. */
344 inp->inp_flags &= ~INP_BOUNDFIB;
345 inp->in6p_laddr = in6addr_any;
346 return (error);
347 }
348 } else {
349 inp->inp_lport = lport;
350 if (in_pcbinshash(inp) != 0) {
351 inp->inp_flags &= ~INP_BOUNDFIB;
352 inp->in6p_laddr = in6addr_any;
353 inp->inp_lport = 0;
354 return (EAGAIN);
355 }
356 }
357 return (0);
358 }
359
360 static int
in6_pcbladdr(struct inpcb * inp,struct sockaddr_in6 * sin6,struct in6_addr * plocal_addr6,bool sas_required)361 in6_pcbladdr(struct inpcb *inp, struct sockaddr_in6 *sin6,
362 struct in6_addr *plocal_addr6, bool sas_required)
363 {
364 int error = 0;
365 int scope_ambiguous = 0;
366 struct in6_addr in6a;
367
368 NET_EPOCH_ASSERT();
369 INP_WLOCK_ASSERT(inp);
370 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); /* XXXRW: why? */
371
372 if (sin6->sin6_port == 0)
373 return (EADDRNOTAVAIL);
374
375 if (sin6->sin6_scope_id == 0 && !V_ip6_use_defzone)
376 scope_ambiguous = 1;
377 if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
378 return(error);
379
380 /* RFC4291 section 2.5.2 */
381 if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
382 return (ENETUNREACH);
383
384 if ((error = prison_remote_ip6(inp->inp_cred, &sin6->sin6_addr)) != 0)
385 return (error);
386
387 if (sas_required) {
388 error = in6_selectsrc_socket(sin6, inp->in6p_outputopts,
389 inp, inp->inp_cred, scope_ambiguous, &in6a, NULL);
390 if (error)
391 return (error);
392 } else {
393 /*
394 * Source address selection isn't required when syncache
395 * has already established connection and both source and
396 * destination addresses was chosen.
397 *
398 * This also includes the case when fwd_tag was used to
399 * select source address in tcp_input().
400 */
401 in6a = inp->in6p_laddr;
402 }
403
404 if (IN6_IS_ADDR_UNSPECIFIED(&in6a))
405 return (EHOSTUNREACH);
406 /*
407 * Do not update this earlier, in case we return with an error.
408 *
409 * XXX: this in6_selectsrc_socket result might replace the bound local
410 * address with the address specified by setsockopt(IPV6_PKTINFO).
411 * Is it the intended behavior?
412 */
413 *plocal_addr6 = in6a;
414
415 /*
416 * Don't do pcblookup call here; return interface in
417 * plocal_addr6
418 * and exit to caller, that will do the lookup.
419 */
420
421 return (0);
422 }
423
424 /*
425 * Outer subroutine:
426 * Connect from a socket to a specified address.
427 * Both address and port must be specified in argument sin.
428 * If don't have a local address for this socket yet,
429 * then pick one.
430 */
431 int
in6_pcbconnect(struct inpcb * inp,struct sockaddr_in6 * sin6,struct ucred * cred,bool sas_required)432 in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred,
433 bool sas_required)
434 {
435 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
436 struct sockaddr_in6 laddr6;
437 int error;
438
439 NET_EPOCH_ASSERT();
440 INP_WLOCK_ASSERT(inp);
441 INP_HASH_WLOCK_ASSERT(pcbinfo);
442 KASSERT(sin6->sin6_family == AF_INET6,
443 ("%s: invalid address family for %p", __func__, sin6));
444 KASSERT(sin6->sin6_len == sizeof(*sin6),
445 ("%s: invalid address length for %p", __func__, sin6));
446 KASSERT(IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr),
447 ("%s: inp is already connected", __func__));
448
449 bzero(&laddr6, sizeof(laddr6));
450 laddr6.sin6_family = AF_INET6;
451
452 #ifdef ROUTE_MPATH
453 if (CALC_FLOWID_OUTBOUND) {
454 uint32_t hash_type, hash_val;
455
456 hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
457 &sin6->sin6_addr, 0, sin6->sin6_port,
458 inp->inp_socket->so_proto->pr_protocol, &hash_type);
459 inp->inp_flowid = hash_val;
460 inp->inp_flowtype = hash_type;
461 }
462 #endif
463 /*
464 * Call inner routine, to assign local interface address.
465 * in6_pcbladdr() may automatically fill in sin6_scope_id.
466 */
467 if ((error = in6_pcbladdr(inp, sin6, &laddr6.sin6_addr,
468 sas_required)) != 0)
469 return (error);
470
471 if (in6_pcblookup_hash_locked(pcbinfo, &sin6->sin6_addr,
472 sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ?
473 &laddr6.sin6_addr : &inp->in6p_laddr, inp->inp_lport, 0,
474 M_NODOM, RT_ALL_FIBS) != NULL)
475 return (EADDRINUSE);
476 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
477 if (inp->inp_lport == 0) {
478 error = in_pcb_lport_dest(inp,
479 (struct sockaddr *) &laddr6, &inp->inp_lport,
480 (struct sockaddr *) sin6, sin6->sin6_port, cred,
481 INPLOOKUP_WILDCARD);
482 if (error)
483 return (error);
484 }
485 inp->in6p_laddr = laddr6.sin6_addr;
486 }
487 inp->in6p_faddr = sin6->sin6_addr;
488 inp->inp_fport = sin6->sin6_port;
489 /* update flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
490 inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
491 if (inp->inp_flags & IN6P_AUTOFLOWLABEL)
492 inp->inp_flow |=
493 (htonl(ip6_randomflowlabel()) & IPV6_FLOWLABEL_MASK);
494
495 if ((inp->inp_flags & INP_INHASHLIST) != 0) {
496 in_pcbrehash(inp);
497 } else {
498 error = in_pcbinshash(inp);
499 MPASS(error == 0);
500 }
501
502 return (0);
503 }
504
505 void
in6_pcbdisconnect(struct inpcb * inp)506 in6_pcbdisconnect(struct inpcb *inp)
507 {
508
509 INP_WLOCK_ASSERT(inp);
510 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
511 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
512 ("%s: inp %p was already disconnected", __func__, inp));
513
514 in_pcbremhash_locked(inp);
515
516 /* See the comment in in_pcbinshash(). */
517 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
518
519 /* XXX-MJ torn writes are visible to SMR lookup */
520 memset(&inp->in6p_laddr, 0, sizeof(inp->in6p_laddr));
521 memset(&inp->in6p_faddr, 0, sizeof(inp->in6p_faddr));
522 inp->inp_fport = 0;
523 /* clear flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
524 inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
525 }
526
527 int
in6_getsockaddr(struct socket * so,struct sockaddr * sa)528 in6_getsockaddr(struct socket *so, struct sockaddr *sa)
529 {
530 struct inpcb *inp;
531
532 inp = sotoinpcb(so);
533 KASSERT(inp != NULL, ("in6_getsockaddr: inp == NULL"));
534
535 *(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
536 .sin6_len = sizeof(struct sockaddr_in6),
537 .sin6_family = AF_INET6,
538 .sin6_port = inp->inp_lport,
539 .sin6_addr = inp->in6p_laddr,
540 };
541 /* XXX: should catch errors */
542 (void)sa6_recoverscope((struct sockaddr_in6 *)sa);
543
544 return (0);
545 }
546
547 int
in6_getpeeraddr(struct socket * so,struct sockaddr * sa)548 in6_getpeeraddr(struct socket *so, struct sockaddr *sa)
549 {
550 struct inpcb *inp;
551
552 inp = sotoinpcb(so);
553 KASSERT(inp != NULL, ("in6_getpeeraddr: inp == NULL"));
554
555 *(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
556 .sin6_len = sizeof(struct sockaddr_in6),
557 .sin6_family = AF_INET6,
558 .sin6_port = inp->inp_fport,
559 .sin6_addr = inp->in6p_faddr,
560 };
561 /* XXX: should catch errors */
562 (void)sa6_recoverscope((struct sockaddr_in6 *)sa);
563
564 return (0);
565 }
566
567 int
in6_mapped_sockaddr(struct socket * so,struct sockaddr * sa)568 in6_mapped_sockaddr(struct socket *so, struct sockaddr *sa)
569 {
570 int error;
571 #ifdef INET
572 struct inpcb *inp;
573
574 inp = sotoinpcb(so);
575 KASSERT(inp != NULL, ("in6_mapped_sockaddr: inp == NULL"));
576
577 if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
578 struct sockaddr_in sin;
579
580 error = in_getsockaddr(so, (struct sockaddr *)&sin);
581 if (error == 0)
582 in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
583 } else
584 #endif
585 {
586 /* scope issues will be handled in in6_getsockaddr(). */
587 error = in6_getsockaddr(so, sa);
588 }
589
590 return error;
591 }
592
593 int
in6_mapped_peeraddr(struct socket * so,struct sockaddr * sa)594 in6_mapped_peeraddr(struct socket *so, struct sockaddr *sa)
595 {
596 int error;
597 #ifdef INET
598 struct inpcb *inp;
599
600 inp = sotoinpcb(so);
601 KASSERT(inp != NULL, ("in6_mapped_peeraddr: inp == NULL"));
602
603 if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
604 struct sockaddr_in sin;
605
606 error = in_getpeeraddr(so, (struct sockaddr *)&sin);
607 if (error == 0)
608 in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
609 } else
610 #endif
611 {
612 /* scope issues will be handled in in6_getpeeraddr(). */
613 error = in6_getpeeraddr(so, sa);
614 }
615
616 return error;
617 }
618
619 /*
620 * Pass some notification to all connections of a protocol
621 * associated with address dst. The local address and/or port numbers
622 * may be specified to limit the search. The "usual action" will be
623 * taken, depending on the ctlinput cmd. The caller must filter any
624 * cmds that are uninteresting (e.g., no error in the map).
625 * Call the protocol specific routine (if any) to report
626 * any errors for each matching socket.
627 */
628 static bool
inp_match6(const struct inpcb * inp,void * v __unused)629 inp_match6(const struct inpcb *inp, void *v __unused)
630 {
631
632 return ((inp->inp_vflag & INP_IPV6) != 0);
633 }
634
635 void
in6_pcbnotify(struct inpcbinfo * pcbinfo,struct sockaddr_in6 * sa6_dst,u_int fport_arg,const struct sockaddr_in6 * src,u_int lport_arg,int errno,void * cmdarg,struct inpcb * (* notify)(struct inpcb *,int))636 in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr_in6 *sa6_dst,
637 u_int fport_arg, const struct sockaddr_in6 *src, u_int lport_arg,
638 int errno, void *cmdarg,
639 struct inpcb *(*notify)(struct inpcb *, int))
640 {
641 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
642 inp_match6, NULL);
643 struct inpcb *inp;
644 struct sockaddr_in6 sa6_src;
645 u_short fport = fport_arg, lport = lport_arg;
646 u_int32_t flowinfo;
647
648 if (IN6_IS_ADDR_UNSPECIFIED(&sa6_dst->sin6_addr))
649 return;
650
651 /*
652 * note that src can be NULL when we get notify by local fragmentation.
653 */
654 sa6_src = (src == NULL) ? sa6_any : *src;
655 flowinfo = sa6_src.sin6_flowinfo;
656
657 while ((inp = inp_next(&inpi)) != NULL) {
658 INP_WLOCK_ASSERT(inp);
659 /*
660 * If the error designates a new path MTU for a destination
661 * and the application (associated with this socket) wanted to
662 * know the value, notify.
663 * XXX: should we avoid to notify the value to TCP sockets?
664 */
665 if (errno == EMSGSIZE && cmdarg != NULL)
666 ip6_notify_pmtu(inp, sa6_dst, *(uint32_t *)cmdarg);
667
668 /*
669 * Detect if we should notify the error. If no source and
670 * destination ports are specified, but non-zero flowinfo and
671 * local address match, notify the error. This is the case
672 * when the error is delivered with an encrypted buffer
673 * by ESP. Otherwise, just compare addresses and ports
674 * as usual.
675 */
676 if (lport == 0 && fport == 0 && flowinfo &&
677 inp->inp_socket != NULL &&
678 flowinfo == (inp->inp_flow & IPV6_FLOWLABEL_MASK) &&
679 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, &sa6_src.sin6_addr))
680 goto do_notify;
681 else if (!IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr,
682 &sa6_dst->sin6_addr) ||
683 inp->inp_socket == 0 ||
684 (lport && inp->inp_lport != lport) ||
685 (!IN6_IS_ADDR_UNSPECIFIED(&sa6_src.sin6_addr) &&
686 !IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr,
687 &sa6_src.sin6_addr)) ||
688 (fport && inp->inp_fport != fport)) {
689 continue;
690 }
691
692 do_notify:
693 if (notify)
694 (*notify)(inp, errno);
695 }
696 }
697
698 /*
699 * Lookup a PCB based on the local address and port. Caller must hold the
700 * hash lock. No inpcb locks or references are acquired.
701 */
702 struct inpcb *
in6_pcblookup_local(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib,int lookupflags,struct ucred * cred)703 in6_pcblookup_local(struct inpcbinfo *pcbinfo, const struct in6_addr *laddr,
704 u_short lport, int fib, int lookupflags, struct ucred *cred)
705 {
706 struct inpcb *inp;
707 int matchwild = 3, wildcard;
708
709 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
710 ("%s: invalid lookup flags %d", __func__, lookupflags));
711 KASSERT(fib == RT_ALL_FIBS || (fib >= 0 && fib < V_rt_numfibs),
712 ("%s: invalid fib %d", __func__, fib));
713
714 INP_HASH_LOCK_ASSERT(pcbinfo);
715
716 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
717 struct inpcbhead *head;
718 /*
719 * Look for an unconnected (wildcard foreign addr) PCB that
720 * matches the local address and port we're looking for.
721 */
722 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
723 pcbinfo->ipi_hashmask)];
724 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
725 /* XXX inp locking */
726 if ((inp->inp_vflag & INP_IPV6) == 0)
727 continue;
728 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) &&
729 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
730 inp->inp_lport == lport && (fib == RT_ALL_FIBS ||
731 inp->inp_inc.inc_fibnum == fib)) {
732 /* Found. */
733 if (prison_equal_ip6(cred->cr_prison,
734 inp->inp_cred->cr_prison))
735 return (inp);
736 }
737 }
738 /*
739 * Not found.
740 */
741 return (NULL);
742 } else {
743 struct inpcbhead *porthash;
744 struct inpcb *match = NULL;
745
746 /*
747 * Port is in use by one or more PCBs. Look for best
748 * fit.
749 */
750 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
751 pcbinfo->ipi_porthashmask)];
752 CK_LIST_FOREACH(inp, porthash, inp_portlist) {
753 if (inp->inp_lport != lport)
754 continue;
755 if (!prison_equal_ip6(cred->cr_prison,
756 inp->inp_cred->cr_prison))
757 continue;
758 /* XXX inp locking */
759 if ((inp->inp_vflag & INP_IPV6) == 0)
760 continue;
761 if (fib != RT_ALL_FIBS &&
762 inp->inp_inc.inc_fibnum != fib)
763 continue;
764 wildcard = 0;
765 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
766 wildcard++;
767 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
768 if (IN6_IS_ADDR_UNSPECIFIED(laddr))
769 wildcard++;
770 else if (!IN6_ARE_ADDR_EQUAL(
771 &inp->in6p_laddr, laddr))
772 continue;
773 } else {
774 if (!IN6_IS_ADDR_UNSPECIFIED(laddr))
775 wildcard++;
776 }
777 if (wildcard < matchwild) {
778 match = inp;
779 matchwild = wildcard;
780 if (matchwild == 0)
781 break;
782 }
783 }
784 return (match);
785 }
786 }
787
788 static bool
in6_multi_match(const struct inpcb * inp,void * v __unused)789 in6_multi_match(const struct inpcb *inp, void *v __unused)
790 {
791
792 if ((inp->inp_vflag & INP_IPV6) && inp->in6p_moptions != NULL)
793 return (true);
794 else
795 return (false);
796 }
797
798 void
in6_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)799 in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
800 {
801 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_RLOCKPCB,
802 in6_multi_match, NULL);
803 struct inpcb *inp;
804 struct in6_multi *inm;
805 struct in6_mfilter *imf;
806 struct ip6_moptions *im6o;
807
808 IN6_MULTI_LOCK_ASSERT();
809
810 while ((inp = inp_next(&inpi)) != NULL) {
811 INP_RLOCK_ASSERT(inp);
812
813 im6o = inp->in6p_moptions;
814 /*
815 * Unselect the outgoing ifp for multicast if it
816 * is being detached.
817 */
818 if (im6o->im6o_multicast_ifp == ifp)
819 im6o->im6o_multicast_ifp = NULL;
820 /*
821 * Drop multicast group membership if we joined
822 * through the interface being detached.
823 */
824 restart:
825 IP6_MFILTER_FOREACH(imf, &im6o->im6o_head) {
826 if ((inm = imf->im6f_in6m) == NULL)
827 continue;
828 if (inm->in6m_ifp != ifp)
829 continue;
830 ip6_mfilter_remove(&im6o->im6o_head, imf);
831 in6_leavegroup_locked(inm, NULL);
832 ip6_mfilter_free(imf);
833 goto restart;
834 }
835 }
836 }
837
838 /*
839 * Check for alternatives when higher level complains
840 * about service problems. For now, invalidate cached
841 * routing information. If the route was created dynamically
842 * (by a redirect), time to try a default gateway again.
843 */
844 void
in6_losing(struct inpcb * inp)845 in6_losing(struct inpcb *inp)
846 {
847
848 RO_INVALIDATE_CACHE(&inp->inp_route6);
849 }
850
851 /*
852 * After a routing change, flush old routing
853 * and allocate a (hopefully) better one.
854 */
855 struct inpcb *
in6_rtchange(struct inpcb * inp,int errno __unused)856 in6_rtchange(struct inpcb *inp, int errno __unused)
857 {
858
859 RO_INVALIDATE_CACHE(&inp->inp_route6);
860 return inp;
861 }
862
863 static bool
in6_pcblookup_lb_match(const struct inpcblbgroup * grp,int domain,int fib)864 in6_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib)
865 {
866 return ((domain == M_NODOM || domain == grp->il_numa_domain) &&
867 (fib == RT_ALL_FIBS || fib == grp->il_fibnum));
868 }
869
870 static struct inpcb *
in6_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,uint16_t fport,const struct in6_addr * laddr,uint16_t lport,uint8_t domain,int fib)871 in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
872 const struct in6_addr *faddr, uint16_t fport, const struct in6_addr *laddr,
873 uint16_t lport, uint8_t domain, int fib)
874 {
875 const struct inpcblbgrouphead *hdr;
876 struct inpcblbgroup *grp;
877 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
878 struct inpcb *inp;
879 u_int count;
880
881 INP_HASH_LOCK_ASSERT(pcbinfo);
882 NET_EPOCH_ASSERT();
883
884 hdr = &pcbinfo->ipi_lbgrouphashbase[
885 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
886
887 /*
888 * Search for an LB group match based on the following criteria:
889 * - prefer jailed groups to non-jailed groups
890 * - prefer exact source address matches to wildcard matches
891 * - prefer groups bound to the specified NUMA domain
892 */
893 jail_exact = jail_wild = local_exact = local_wild = NULL;
894 CK_LIST_FOREACH(grp, hdr, il_list) {
895 bool injail;
896
897 #ifdef INET
898 if (!(grp->il_vflag & INP_IPV6))
899 continue;
900 #endif
901 if (grp->il_lport != lport)
902 continue;
903
904 injail = prison_flag(grp->il_cred, PR_IP6) != 0;
905 if (injail && prison_check_ip6_locked(grp->il_cred->cr_prison,
906 laddr) != 0)
907 continue;
908
909 if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) {
910 if (injail) {
911 jail_exact = grp;
912 if (in6_pcblookup_lb_match(grp, domain, fib))
913 /* This is a perfect match. */
914 goto out;
915 } else if (local_exact == NULL ||
916 in6_pcblookup_lb_match(grp, domain, fib)) {
917 local_exact = grp;
918 }
919 } else if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr)) {
920 if (injail) {
921 if (jail_wild == NULL ||
922 in6_pcblookup_lb_match(grp, domain, fib))
923 jail_wild = grp;
924 } else if (local_wild == NULL ||
925 in6_pcblookup_lb_match(grp, domain, fib)) {
926 local_wild = grp;
927 }
928 }
929 }
930
931 if (jail_exact != NULL)
932 grp = jail_exact;
933 else if (jail_wild != NULL)
934 grp = jail_wild;
935 else if (local_exact != NULL)
936 grp = local_exact;
937 else
938 grp = local_wild;
939 if (grp == NULL)
940 return (NULL);
941 out:
942 /*
943 * Synchronize with in_pcblbgroup_insert().
944 */
945 count = atomic_load_acq_int(&grp->il_inpcnt);
946 if (count == 0)
947 return (NULL);
948 inp = grp->il_inp[INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
949 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
950 return (inp);
951 }
952
953 static bool
in6_pcblookup_exact_match(const struct inpcb * inp,const struct in6_addr * faddr,u_short fport,const struct in6_addr * laddr,u_short lport)954 in6_pcblookup_exact_match(const struct inpcb *inp, const struct in6_addr *faddr,
955 u_short fport, const struct in6_addr *laddr, u_short lport)
956 {
957 /* XXX inp locking */
958 if ((inp->inp_vflag & INP_IPV6) == 0)
959 return (false);
960 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, faddr) &&
961 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
962 inp->inp_fport == fport && inp->inp_lport == lport)
963 return (true);
964 return (false);
965 }
966
967 static struct inpcb *
in6_pcblookup_hash_exact(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_short fport,const struct in6_addr * laddr,u_short lport)968 in6_pcblookup_hash_exact(struct inpcbinfo *pcbinfo,
969 const struct in6_addr *faddr, u_short fport,
970 const struct in6_addr *laddr, u_short lport)
971 {
972 struct inpcbhead *head;
973 struct inpcb *inp;
974
975 INP_HASH_LOCK_ASSERT(pcbinfo);
976
977 /*
978 * First look for an exact match.
979 */
980 head = &pcbinfo->ipi_hash_exact[INP6_PCBHASH(faddr, lport, fport,
981 pcbinfo->ipi_hashmask)];
982 CK_LIST_FOREACH(inp, head, inp_hash_exact) {
983 if (in6_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
984 return (inp);
985 }
986 return (NULL);
987 }
988
989 typedef enum {
990 INPLOOKUP_MATCH_NONE = 0,
991 INPLOOKUP_MATCH_WILD = 1,
992 INPLOOKUP_MATCH_LADDR = 2,
993 } inp_lookup_match_t;
994
995 static inp_lookup_match_t
in6_pcblookup_wild_match(const struct inpcb * inp,const struct in6_addr * laddr,u_short lport,int fib)996 in6_pcblookup_wild_match(const struct inpcb *inp, const struct in6_addr *laddr,
997 u_short lport, int fib)
998 {
999 /* XXX inp locking */
1000 if ((inp->inp_vflag & INP_IPV6) == 0)
1001 return (INPLOOKUP_MATCH_NONE);
1002 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) ||
1003 inp->inp_lport != lport)
1004 return (INPLOOKUP_MATCH_NONE);
1005 if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib)
1006 return (INPLOOKUP_MATCH_NONE);
1007 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
1008 return (INPLOOKUP_MATCH_WILD);
1009 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr))
1010 return (INPLOOKUP_MATCH_LADDR);
1011 return (INPLOOKUP_MATCH_NONE);
1012 }
1013
1014 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
1015
1016 static struct inpcb *
in6_pcblookup_hash_wild_smr(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib,const inp_lookup_t lockflags)1017 in6_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo,
1018 const struct in6_addr *laddr, u_short lport, int fib,
1019 const inp_lookup_t lockflags)
1020 {
1021 struct inpcbhead *head;
1022 struct inpcb *inp;
1023
1024 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
1025 ("%s: not in SMR read section", __func__));
1026
1027 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1028 pcbinfo->ipi_hashmask)];
1029 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1030 inp_lookup_match_t match;
1031
1032 match = in6_pcblookup_wild_match(inp, laddr, lport, fib);
1033 if (match == INPLOOKUP_MATCH_NONE)
1034 continue;
1035
1036 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1037 match = in6_pcblookup_wild_match(inp, laddr, lport,
1038 fib);
1039 if (match != INPLOOKUP_MATCH_NONE &&
1040 prison_check_ip6_locked(inp->inp_cred->cr_prison,
1041 laddr) == 0)
1042 return (inp);
1043 inp_unlock(inp, lockflags);
1044 }
1045
1046 /*
1047 * The matching socket disappeared out from under us. Fall back
1048 * to a serialized lookup.
1049 */
1050 return (INP_LOOKUP_AGAIN);
1051 }
1052 return (NULL);
1053 }
1054
1055 static struct inpcb *
in6_pcblookup_hash_wild_locked(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib)1056 in6_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo,
1057 const struct in6_addr *laddr, u_short lport, int fib)
1058 {
1059 struct inpcbhead *head;
1060 struct inpcb *inp, *jail_wild, *local_exact, *local_wild;
1061
1062 INP_HASH_LOCK_ASSERT(pcbinfo);
1063
1064 /*
1065 * Order of socket selection - we always prefer jails.
1066 * 1. jailed, non-wild.
1067 * 2. jailed, wild.
1068 * 3. non-jailed, non-wild.
1069 * 4. non-jailed, wild.
1070 */
1071 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1072 pcbinfo->ipi_hashmask)];
1073 local_wild = local_exact = jail_wild = NULL;
1074 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1075 inp_lookup_match_t match;
1076 bool injail;
1077
1078 match = in6_pcblookup_wild_match(inp, laddr, lport, fib);
1079 if (match == INPLOOKUP_MATCH_NONE)
1080 continue;
1081
1082 injail = prison_flag(inp->inp_cred, PR_IP6) != 0;
1083 if (injail) {
1084 if (prison_check_ip6_locked(
1085 inp->inp_cred->cr_prison, laddr) != 0)
1086 continue;
1087 } else {
1088 if (local_exact != NULL)
1089 continue;
1090 }
1091
1092 if (match == INPLOOKUP_MATCH_LADDR) {
1093 if (injail)
1094 return (inp);
1095 else
1096 local_exact = inp;
1097 } else {
1098 if (injail)
1099 jail_wild = inp;
1100 else
1101 local_wild = inp;
1102 }
1103 }
1104
1105 if (jail_wild != NULL)
1106 return (jail_wild);
1107 if (local_exact != NULL)
1108 return (local_exact);
1109 if (local_wild != NULL)
1110 return (local_wild);
1111 return (NULL);
1112 }
1113
1114 struct inpcb *
in6_pcblookup_hash_locked(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport_arg,const struct in6_addr * laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)1115 in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
1116 const struct in6_addr *faddr, u_int fport_arg,
1117 const struct in6_addr *laddr, u_int lport_arg,
1118 int lookupflags, uint8_t numa_domain, int fib)
1119 {
1120 struct inpcb *inp;
1121 u_short fport = fport_arg, lport = lport_arg;
1122
1123 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0,
1124 ("%s: invalid lookup flags %d", __func__, lookupflags));
1125 KASSERT(!IN6_IS_ADDR_UNSPECIFIED(faddr),
1126 ("%s: invalid foreign address", __func__));
1127 KASSERT(!IN6_IS_ADDR_UNSPECIFIED(laddr),
1128 ("%s: invalid local address", __func__));
1129 INP_HASH_LOCK_ASSERT(pcbinfo);
1130
1131 inp = in6_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
1132 if (inp != NULL)
1133 return (inp);
1134
1135 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1136 inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport, laddr,
1137 lport, numa_domain, fib);
1138 if (inp == NULL) {
1139 inp = in6_pcblookup_hash_wild_locked(pcbinfo,
1140 laddr, lport, fib);
1141 }
1142 }
1143 return (inp);
1144 }
1145
1146 static struct inpcb *
in6_pcblookup_hash(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,uint8_t numa_domain,int fib)1147 in6_pcblookup_hash(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1148 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1149 uint8_t numa_domain, int fib)
1150 {
1151 struct inpcb *inp;
1152 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
1153
1154 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1155 ("%s: LOCKPCB not set", __func__));
1156
1157 INP_HASH_WLOCK(pcbinfo);
1158 inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1159 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
1160 if (inp != NULL && !inp_trylock(inp, lockflags)) {
1161 in_pcbref(inp);
1162 INP_HASH_WUNLOCK(pcbinfo);
1163 inp_lock(inp, lockflags);
1164 if (in_pcbrele(inp, lockflags))
1165 /* XXX-MJ or retry until we get a negative match? */
1166 inp = NULL;
1167 } else {
1168 INP_HASH_WUNLOCK(pcbinfo);
1169 }
1170 return (inp);
1171 }
1172
1173 static struct inpcb *
in6_pcblookup_hash_smr(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport_arg,const struct in6_addr * laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)1174 in6_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1175 u_int fport_arg, const struct in6_addr *laddr, u_int lport_arg,
1176 int lookupflags, uint8_t numa_domain, int fib)
1177 {
1178 struct inpcb *inp;
1179 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
1180 const u_short fport = fport_arg, lport = lport_arg;
1181
1182 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1183 ("%s: invalid lookup flags %d", __func__, lookupflags));
1184 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1185 ("%s: LOCKPCB not set", __func__));
1186
1187 smr_enter(pcbinfo->ipi_smr);
1188 inp = in6_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
1189 if (inp != NULL) {
1190 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1191 if (__predict_true(in6_pcblookup_exact_match(inp,
1192 faddr, fport, laddr, lport)))
1193 return (inp);
1194 inp_unlock(inp, lockflags);
1195 }
1196 /*
1197 * We failed to lock the inpcb, or its connection state changed
1198 * out from under us. Fall back to a precise search.
1199 */
1200 return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
1201 lookupflags, numa_domain, fib));
1202 }
1203
1204 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1205 inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport,
1206 laddr, lport, numa_domain, fib);
1207 if (inp != NULL) {
1208 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1209 if (__predict_true(in6_pcblookup_wild_match(inp,
1210 laddr, lport, fib) != INPLOOKUP_MATCH_NONE))
1211 return (inp);
1212 inp_unlock(inp, lockflags);
1213 }
1214 inp = INP_LOOKUP_AGAIN;
1215 } else {
1216 inp = in6_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,
1217 fib, lockflags);
1218 }
1219 if (inp == INP_LOOKUP_AGAIN) {
1220 return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr,
1221 lport, lookupflags, numa_domain, fib));
1222 }
1223 }
1224
1225 if (inp == NULL)
1226 smr_exit(pcbinfo->ipi_smr);
1227
1228 return (inp);
1229 }
1230
1231 /*
1232 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1233 * from which a pre-calculated hash value may be extracted.
1234 */
1235 struct inpcb *
in6_pcblookup(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,struct ifnet * ifp)1236 in6_pcblookup(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1237 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1238 struct ifnet *ifp)
1239 {
1240 int fib;
1241
1242 fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS;
1243 return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
1244 lookupflags, M_NODOM, fib));
1245 }
1246
1247 struct inpcb *
in6_pcblookup_mbuf(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)1248 in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1249 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1250 struct ifnet *ifp __unused, struct mbuf *m)
1251 {
1252 int fib;
1253
1254 M_ASSERTPKTHDR(m);
1255 fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS;
1256 return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
1257 lookupflags, m->m_pkthdr.numa_domain, fib));
1258 }
1259
1260 void
init_sin6(struct sockaddr_in6 * sin6,struct mbuf * m,int srcordst)1261 init_sin6(struct sockaddr_in6 *sin6, struct mbuf *m, int srcordst)
1262 {
1263 struct ip6_hdr *ip;
1264
1265 ip = mtod(m, struct ip6_hdr *);
1266 bzero(sin6, sizeof(*sin6));
1267 sin6->sin6_len = sizeof(*sin6);
1268 sin6->sin6_family = AF_INET6;
1269 sin6->sin6_addr = srcordst ? ip->ip6_dst : ip->ip6_src;
1270
1271 (void)sa6_recoverscope(sin6); /* XXX: should catch errors... */
1272
1273 return;
1274 }
1275