1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <np@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33
34 #ifdef TCP_OFFLOAD
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/module.h>
40 #include <sys/protosw.h>
41 #include <sys/domain.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/sysctl.h>
45 #include <net/ethernet.h>
46 #include <net/if.h>
47 #include <net/if_types.h>
48 #include <net/if_vlan_var.h>
49 #include <net/route.h>
50 #include <net/route/nhop.h>
51 #include <netinet/in.h>
52 #include <netinet/in_pcb.h>
53 #include <netinet/ip.h>
54 #define TCPSTATES
55 #include <netinet/tcp_fsm.h>
56 #include <netinet/tcp_var.h>
57 #include <netinet/toecore.h>
58 #include <netinet/cc/cc.h>
59
60 #include "common/common.h"
61 #include "common/t4_msg.h"
62 #include "common/t4_regs.h"
63 #include "common/t4_regs_values.h"
64 #include "t4_clip.h"
65 #include "tom/t4_tom_l2t.h"
66 #include "tom/t4_tom.h"
67
68 /*
69 * Active open succeeded.
70 */
71 static int
do_act_establish(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)72 do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
73 struct mbuf *m)
74 {
75 struct adapter *sc = iq->adapter;
76 const struct cpl_act_establish *cpl = (const void *)(rss + 1);
77 u_int tid = GET_TID(cpl);
78 u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
79 struct toepcb *toep = lookup_atid(sc, atid);
80 struct inpcb *inp = toep->inp;
81
82 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
83 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
84
85 CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid);
86 free_atid(sc, atid);
87
88 CURVNET_SET(toep->vnet);
89 INP_WLOCK(inp);
90 toep->tid = tid;
91 insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1);
92 if (sc->params.tid_qid_sel_mask != 0) {
93 update_tid_qid_sel(toep->vi, &toep->params, tid);
94 toep->ofld_txq = &sc->sge.ofld_txq[toep->params.txq_idx];
95 toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx];
96 }
97
98 if (inp->inp_flags & INP_DROPPED) {
99
100 /* socket closed by the kernel before hw told us it connected */
101
102 send_flowc_wr(toep, NULL);
103 send_reset(sc, toep, be32toh(cpl->snd_isn));
104 goto done;
105 }
106
107 make_established(toep, be32toh(cpl->snd_isn) - 1,
108 be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt);
109 inp->inp_flowtype = M_HASHTYPE_OPAQUE;
110 inp->inp_flowid = tid;
111
112 done:
113 INP_WUNLOCK(inp);
114 CURVNET_RESTORE();
115 return (0);
116 }
117
118 void
act_open_failure_cleanup(struct adapter * sc,struct toepcb * toep,u_int status)119 act_open_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status)
120 {
121 struct inpcb *inp = toep->inp;
122 struct toedev *tod = &toep->td->tod;
123 struct epoch_tracker et;
124 struct tom_data *td = sc->tom_softc;
125
126 if (toep->tid >= 0) {
127 free_atid(sc, toep->tid);
128 toep->tid = -1;
129 mtx_lock(&td->toep_list_lock);
130 if (toep->flags & TPF_IN_TOEP_LIST) {
131 toep->flags &= ~TPF_IN_TOEP_LIST;
132 TAILQ_REMOVE(&td->toep_list, toep, link);
133 }
134 mtx_unlock(&td->toep_list_lock);
135 }
136
137 CURVNET_SET(toep->vnet);
138 if (status != EAGAIN)
139 NET_EPOCH_ENTER(et);
140 INP_WLOCK(inp);
141 toe_connect_failed(tod, inp, status);
142 final_cpl_received(toep); /* unlocks inp */
143 if (status != EAGAIN)
144 NET_EPOCH_EXIT(et);
145 CURVNET_RESTORE();
146 }
147
148 /*
149 * Active open failed.
150 */
151 static int
do_act_open_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)152 do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
153 struct mbuf *m)
154 {
155 struct adapter *sc = iq->adapter;
156 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
157 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
158 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
159 struct toepcb *toep = lookup_atid(sc, atid);
160 int rc;
161
162 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
163 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
164
165 CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
166
167 /* Ignore negative advice */
168 if (negative_advice(status))
169 return (0);
170
171 if (status && act_open_has_tid(status))
172 release_tid(sc, GET_TID(cpl), toep->ctrlq);
173
174 rc = act_open_rpl_status_to_errno(status);
175 act_open_failure_cleanup(sc, toep, rc);
176
177 return (0);
178 }
179
180 void
t4_init_connect_cpl_handlers(void)181 t4_init_connect_cpl_handlers(void)
182 {
183
184 t4_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
185 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl,
186 CPL_COOKIE_TOM);
187 }
188
189 void
t4_uninit_connect_cpl_handlers(void)190 t4_uninit_connect_cpl_handlers(void)
191 {
192
193 t4_register_cpl_handler(CPL_ACT_ESTABLISH, NULL);
194 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, CPL_COOKIE_TOM);
195 }
196
197 #ifdef KTR
198 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
199 reason = __LINE__; \
200 rc = (x); \
201 goto failed; \
202 } while (0)
203 #else
204 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
205 rc = (x); \
206 goto failed; \
207 } while (0)
208 #endif
209
210 static inline int
act_open_cpl_size(struct adapter * sc,int isipv6)211 act_open_cpl_size(struct adapter *sc, int isipv6)
212 {
213 int idx;
214 static const int sz_table[4][2] = {
215 {
216 sizeof (struct cpl_act_open_req),
217 sizeof (struct cpl_act_open_req6)
218 },
219 {
220 sizeof (struct cpl_t5_act_open_req),
221 sizeof (struct cpl_t5_act_open_req6)
222 },
223 {
224 sizeof (struct cpl_t6_act_open_req),
225 sizeof (struct cpl_t6_act_open_req6)
226 },
227 {
228 sizeof (struct cpl_t7_act_open_req),
229 sizeof (struct cpl_t7_act_open_req6)
230 },
231 };
232
233 MPASS(chip_id(sc) >= CHELSIO_T4);
234 idx = min(chip_id(sc) - CHELSIO_T4, 3);
235
236 return (sz_table[idx][!!isipv6]);
237 }
238
239 /*
240 * active open (soconnect).
241 *
242 * State of affairs on entry:
243 * soisconnecting (so_state |= SS_ISCONNECTING)
244 * tcbinfo not locked (This has changed - used to be WLOCKed)
245 * inp WLOCKed
246 * tp->t_state = TCPS_SYN_SENT
247 * rtalloc1, RT_UNLOCK on rt.
248 */
249 int
t4_connect(struct toedev * tod,struct socket * so,struct nhop_object * nh,struct sockaddr * nam)250 t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh,
251 struct sockaddr *nam)
252 {
253 struct adapter *sc = tod->tod_softc;
254 struct tom_data *td;
255 struct toepcb *toep = NULL;
256 struct wrqe *wr = NULL;
257 if_t rt_ifp = nh->nh_ifp;
258 struct vi_info *vi;
259 int qid_atid, rc, isipv6;
260 struct inpcb *inp = sotoinpcb(so);
261 struct tcpcb *tp = intotcpcb(inp);
262 #ifdef KTR
263 int reason;
264 #endif
265 struct offload_settings settings;
266 struct epoch_tracker et;
267 uint16_t vid = 0xfff, pcp = 0;
268 uint64_t ntuple;
269
270 INP_WLOCK_ASSERT(inp);
271 KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
272 ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
273
274 if (if_gettype(rt_ifp) == IFT_ETHER)
275 vi = if_getsoftc(rt_ifp);
276 else if (if_gettype(rt_ifp) == IFT_L2VLAN) {
277 if_t ifp = VLAN_TRUNKDEV(rt_ifp);
278
279 vi = if_getsoftc(ifp);
280 VLAN_TAG(rt_ifp, &vid);
281 VLAN_PCP(rt_ifp, &pcp);
282 } else if (if_gettype(rt_ifp) == IFT_IEEE8023ADLAG)
283 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */
284 else
285 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
286 if (sc->flags & KERN_TLS_ON)
287 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
288
289 rw_rlock(&sc->policy_lock);
290 settings = *lookup_offload_policy(sc, OPEN_TYPE_ACTIVE, NULL,
291 EVL_MAKETAG(vid, pcp, 0), inp);
292 rw_runlock(&sc->policy_lock);
293 if (!settings.offload)
294 DONT_OFFLOAD_ACTIVE_OPEN(EPERM);
295
296 toep = alloc_toepcb(vi, M_NOWAIT);
297 if (toep == NULL)
298 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
299
300 toep->tid = alloc_atid(sc, toep);
301 if (toep->tid < 0)
302 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
303
304 toep->l2te = t4_l2t_get(vi->pi, rt_ifp,
305 nh->nh_flags & NHF_GATEWAY ? &nh->gw_sa : nam);
306 if (toep->l2te == NULL)
307 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
308
309 toep->vnet = so->so_vnet;
310 init_conn_params(vi, &settings, &inp->inp_inc, so, NULL,
311 toep->l2te->idx, &toep->params);
312 init_toepcb(vi, toep);
313
314 isipv6 = nam->sa_family == AF_INET6;
315 wr = alloc_wrqe(act_open_cpl_size(sc, isipv6), toep->ctrlq);
316 if (wr == NULL)
317 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
318
319 qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) |
320 V_TID_COOKIE(CPL_COOKIE_TOM);
321
322 ntuple = select_ntuple(vi, toep->l2te);
323 if (isipv6) {
324 struct cpl_act_open_req6 *cpl = wrtod(wr);
325 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
326 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
327 struct cpl_t7_act_open_req6 *cpl7 = (void *)cpl;
328
329 if ((inp->inp_vflag & INP_IPV6) == 0)
330 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
331
332 toep->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
333 if (toep->ce == NULL)
334 DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
335
336 switch (chip_id(sc)) {
337 case CHELSIO_T4:
338 INIT_TP_WR(cpl, 0);
339 cpl->params = htobe32((uint32_t)ntuple);
340 break;
341 case CHELSIO_T5:
342 INIT_TP_WR(cpl5, 0);
343 cpl5->iss = htobe32(tp->iss);
344 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple));
345 break;
346 case CHELSIO_T6:
347 INIT_TP_WR(cpl6, 0);
348 cpl6->iss = htobe32(tp->iss);
349 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple));
350 break;
351 case CHELSIO_T7:
352 default:
353 INIT_TP_WR(cpl7, 0);
354 cpl7->iss = htobe32(tp->iss);
355 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple));
356 break;
357 }
358 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
359 qid_atid));
360 cpl->local_port = inp->inp_lport;
361 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
362 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
363 cpl->peer_port = inp->inp_fport;
364 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
365 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
366 cpl->opt0 = calc_options0(vi, &toep->params);
367 cpl->opt2 = calc_options2(vi, &toep->params);
368
369 CTR6(KTR_CXGBE,
370 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
371 __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
372 be32toh(cpl->opt2));
373 } else {
374 struct cpl_act_open_req *cpl = wrtod(wr);
375 struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
376 struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
377 struct cpl_t7_act_open_req *cpl7 = (void *)cpl;
378
379 switch (chip_id(sc)) {
380 case CHELSIO_T4:
381 INIT_TP_WR(cpl, 0);
382 cpl->params = htobe32((uint32_t)ntuple);
383 break;
384 case CHELSIO_T5:
385 INIT_TP_WR(cpl5, 0);
386 cpl5->iss = htobe32(tp->iss);
387 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple));
388 break;
389 case CHELSIO_T6:
390 INIT_TP_WR(cpl6, 0);
391 cpl6->iss = htobe32(tp->iss);
392 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple));
393 break;
394 case CHELSIO_T7:
395 default:
396 INIT_TP_WR(cpl7, 0);
397 cpl7->iss = htobe32(tp->iss);
398 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple));
399 }
400 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
401 qid_atid));
402 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
403 &cpl->peer_ip, &cpl->peer_port);
404 cpl->opt0 = calc_options0(vi, &toep->params);
405 cpl->opt2 = calc_options2(vi, &toep->params);
406
407 CTR6(KTR_CXGBE,
408 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
409 __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
410 be32toh(cpl->opt2));
411 }
412
413 offload_socket(so, toep);
414 /* Add the TOE PCB to the active list */
415 td = toep->td;
416 mtx_lock(&td->toep_list_lock);
417 TAILQ_INSERT_TAIL(&td->toep_list, toep, link);
418 toep->flags |= TPF_IN_TOEP_LIST;
419 mtx_unlock(&td->toep_list_lock);
420 NET_EPOCH_ENTER(et);
421 rc = t4_l2t_send(sc, wr, toep->l2te);
422 NET_EPOCH_EXIT(et);
423 if (rc == 0) {
424 toep->flags |= TPF_CPL_PENDING;
425 return (0);
426 }
427
428 undo_offload_socket(so);
429 #if defined(KTR)
430 reason = __LINE__;
431 #endif
432 failed:
433 CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc);
434
435 if (wr)
436 free_wrqe(wr);
437
438 if (toep) {
439 if (toep->tid >= 0)
440 free_atid(sc, toep->tid);
441 if (toep->l2te)
442 t4_l2t_release(toep->l2te);
443 if (toep->ce)
444 t4_release_clip_entry(sc, toep->ce);
445 free_toepcb(toep);
446 }
447
448 return (rc);
449 }
450 #endif
451