Lines Matching full:subflow
70 struct mptcp_subflow_context *subflow; in __mptcp_socket_create() local
81 subflow = mptcp_subflow_ctx(ssock->sk); in __mptcp_socket_create()
82 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
84 subflow->request_mptcp = 1; in __mptcp_socket_create()
85 subflow->subflow_id = msk->subflow_id++; in __mptcp_socket_create()
87 /* This is the first subflow, always with id 0 */ in __mptcp_socket_create()
88 WRITE_ONCE(subflow->local_id, 0); in __mptcp_socket_create()
95 /* If the MPC handshake is not started, returns the first subflow,
277 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skb() local
287 /* try to fetch required memory from subflow */ in __mptcp_move_skb()
299 MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); in __mptcp_move_skb()
420 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) in mptcp_timeout_from_subflow() argument
422 const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_timeout_from_subflow()
424 return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? in mptcp_timeout_from_subflow()
430 struct mptcp_subflow_context *subflow; in mptcp_set_timeout() local
433 mptcp_for_each_subflow(mptcp_sk(sk), subflow) in mptcp_set_timeout()
434 tout = max(tout, mptcp_timeout_from_subflow(subflow)); in mptcp_set_timeout()
461 struct mptcp_subflow_context *subflow; in mptcp_send_ack() local
463 mptcp_for_each_subflow(msk, subflow) in mptcp_send_ack()
464 mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); in mptcp_send_ack()
493 struct mptcp_subflow_context *subflow; in mptcp_cleanup_rbuf() local
501 mptcp_for_each_subflow(msk, subflow) { in mptcp_cleanup_rbuf()
502 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_cleanup_rbuf()
518 * at the subflow level and the msk lock was not held, so this in mptcp_check_data_fin()
574 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skbs_from_subflow() local
592 map_remaining = subflow->map_data_len - in __mptcp_move_skbs_from_subflow()
593 mptcp_subflow_get_map_offset(subflow); in __mptcp_move_skbs_from_subflow()
605 subflow->map_data_len = skb->len; in __mptcp_move_skbs_from_subflow()
719 struct mptcp_subflow_context *subflow; in __mptcp_error_report() local
722 mptcp_for_each_subflow(msk, subflow) in __mptcp_error_report()
723 if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) in __mptcp_error_report()
773 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_data_ready() local
776 * subflow at msk destruction time, but we must avoid enqueuing in mptcp_data_ready()
779 if (unlikely(subflow->disposable)) in mptcp_data_ready()
820 struct mptcp_subflow_context *tmp, *subflow; in __mptcp_flush_join_list() local
823 list_for_each_entry_safe(subflow, tmp, join_list, node) { in __mptcp_flush_join_list()
824 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_flush_join_list()
827 list_move_tail(&subflow->node, &msk->conn_list); in __mptcp_flush_join_list()
856 /* each subflow already holds a reference to the sk, and the in mptcp_schedule_work()
857 * workqueue is invoked by a subflow, so sk can't go away here. in mptcp_schedule_work()
986 struct mptcp_subflow_context *subflow; in mptcp_enter_memory_pressure() local
990 mptcp_for_each_subflow(msk, subflow) { in mptcp_enter_memory_pressure()
991 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_enter_memory_pressure()
1294 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) in mptcp_subflow_set_active() argument
1296 if (!subflow->stale) in mptcp_subflow_set_active()
1299 subflow->stale = 0; in mptcp_subflow_set_active()
1300 MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); in mptcp_subflow_set_active()
1303 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) in mptcp_subflow_active() argument
1305 if (unlikely(subflow->stale)) { in mptcp_subflow_active()
1306 u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); in mptcp_subflow_active()
1308 if (subflow->stale_rcv_tstamp == rcv_tstamp) in mptcp_subflow_active()
1311 mptcp_subflow_set_active(subflow); in mptcp_subflow_active()
1313 return __mptcp_subflow_active(subflow); in mptcp_subflow_active()
1321 * returns the subflow that will transmit the next DSS
1327 struct mptcp_subflow_context *subflow; in mptcp_subflow_get_send() local
1335 /* pick the subflow with the lower wmem/wspace ratio */ in mptcp_subflow_get_send()
1341 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1342 bool backup = subflow->backup || subflow->request_bkup; in mptcp_subflow_get_send()
1344 trace_mptcp_subflow_get_send(subflow); in mptcp_subflow_get_send()
1345 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_send()
1346 if (!mptcp_subflow_active(subflow)) in mptcp_subflow_get_send()
1349 tout = max(tout, mptcp_timeout_from_subflow(subflow)); in mptcp_subflow_get_send()
1351 pace = subflow->avg_pacing_rate; in mptcp_subflow_get_send()
1354 subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); in mptcp_subflow_get_send()
1355 pace = subflow->avg_pacing_rate; in mptcp_subflow_get_send()
1368 /* pick the best backup if no other subflow is active */ in mptcp_subflow_get_send()
1378 * otherwise do not use the picked, slower, subflow in mptcp_subflow_get_send()
1379 * We select the subflow with the shorter estimated time to flush in mptcp_subflow_get_send()
1381 * to check that subflow has a non empty cwin. in mptcp_subflow_get_send()
1392 subflow = mptcp_subflow_ctx(ssk); in mptcp_subflow_get_send()
1393 subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + in mptcp_subflow_get_send()
1424 * that has been handed to the subflow for transmission in mptcp_update_post_push()
1497 struct mptcp_subflow_context *subflow; in __mptcp_push_pending() local
1505 mptcp_for_each_subflow(msk, subflow) { in __mptcp_push_pending()
1506 if (READ_ONCE(subflow->scheduled)) { in __mptcp_push_pending()
1507 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_push_pending()
1510 ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_push_pending()
1518 /* Need to lock the new subflow only if different in __mptcp_push_pending()
1540 /* at this point we held the socket lock for the last subflow we used */ in __mptcp_push_pending()
1563 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_subflow_push_pending() local
1566 /* check for a different subflow usage only after in __mptcp_subflow_push_pending()
1570 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_subflow_push_pending()
1582 if (READ_ONCE(subflow->scheduled)) { in __mptcp_subflow_push_pending()
1583 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_subflow_push_pending()
1590 mptcp_for_each_subflow(msk, subflow) { in __mptcp_subflow_push_pending()
1591 if (READ_ONCE(subflow->scheduled)) { in __mptcp_subflow_push_pending()
1592 xmit_ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_subflow_push_pending()
1594 mptcp_subflow_delegate(subflow, in __mptcp_subflow_push_pending()
1629 * first subflow right now. Otherwise we are in the defer_connect in mptcp_sendmsg_fastopen()
1630 * path, and the first subflow must be already present. in mptcp_sendmsg_fastopen()
1632 * fastopen attempt, no need to check for additional subflow status. in mptcp_sendmsg_fastopen()
1910 struct mptcp_subflow_context *subflow; in mptcp_rcv_space_adjust() local
1934 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1939 tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); in mptcp_rcv_space_adjust()
1979 * get drops at subflow level if skbs can't be moved to in mptcp_rcv_space_adjust()
1983 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1987 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_rcv_space_adjust()
2006 struct mptcp_subflow_context *subflow) in __mptcp_first_ready_from() argument
2008 struct mptcp_subflow_context *start_subflow = subflow; in __mptcp_first_ready_from()
2010 while (!READ_ONCE(subflow->data_avail)) { in __mptcp_first_ready_from()
2011 subflow = mptcp_next_subflow(msk, subflow); in __mptcp_first_ready_from()
2012 if (subflow == start_subflow) in __mptcp_first_ready_from()
2015 return subflow; in __mptcp_first_ready_from()
2020 struct mptcp_subflow_context *subflow; in __mptcp_move_skbs() local
2027 /* verify we can move any data from the subflow, eventually updating */ in __mptcp_move_skbs()
2029 mptcp_for_each_subflow(msk, subflow) in __mptcp_move_skbs()
2030 __mptcp_rcvbuf_update(sk, subflow->tcp_sock); in __mptcp_move_skbs()
2032 subflow = list_first_entry(&msk->conn_list, in __mptcp_move_skbs()
2040 * and ev. acquiring the subflow socket lock before baling out in __mptcp_move_skbs()
2045 subflow = __mptcp_first_ready_from(msk, subflow); in __mptcp_move_skbs()
2046 if (!subflow) in __mptcp_move_skbs()
2049 ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_move_skbs()
2056 subflow = mptcp_next_subflow(msk, subflow); in __mptcp_move_skbs()
2230 /* Find an idle subflow. Return NULL if there is unacked data at tcp
2233 * A backup subflow is returned only if that is the only kind available.
2238 struct mptcp_subflow_context *subflow; in mptcp_subflow_get_retrans() local
2241 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
2242 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_retrans()
2244 if (!__mptcp_subflow_active(subflow)) in mptcp_subflow_get_retrans()
2250 min_stale_count = min_t(int, min_stale_count, subflow->stale_count); in mptcp_subflow_get_retrans()
2254 if (subflow->backup || subflow->request_bkup) { in mptcp_subflow_get_retrans()
2313 * clean completely the subflow status when the subflow reaches
2317 struct mptcp_subflow_context *subflow, in __mptcp_subflow_disconnect() argument
2322 /* The MPTCP code never wait on the subflow sockets, TCP-level in __mptcp_subflow_disconnect()
2326 mptcp_subflow_ctx_reset(subflow); in __mptcp_subflow_disconnect()
2332 /* subflow sockets can be either outgoing (connect) or incoming
2341 struct mptcp_subflow_context *subflow, in __mptcp_close_ssk() argument
2347 /* If the first subflow moved to a close state before accept, e.g. due in __mptcp_close_ssk()
2348 * to an incoming reset or listener shutdown, the subflow socket is in __mptcp_close_ssk()
2364 list_del(&subflow->node); in __mptcp_close_ssk()
2374 subflow->send_fastclose = 1; in __mptcp_close_ssk()
2379 __mptcp_subflow_disconnect(ssk, subflow, flags); in __mptcp_close_ssk()
2385 subflow->disposable = 1; in __mptcp_close_ssk()
2393 kfree_rcu(subflow, rcu); in __mptcp_close_ssk()
2395 /* otherwise tcp will dispose of the ssk and subflow ctx */ in __mptcp_close_ssk()
2434 struct mptcp_subflow_context *subflow) in mptcp_close_ssk() argument
2436 /* The first subflow can already be closed and still in the list */ in mptcp_close_ssk()
2437 if (subflow->close_event_done) in mptcp_close_ssk()
2440 subflow->close_event_done = true; in mptcp_close_ssk()
2445 /* subflow aborted before reaching the fully_established status in mptcp_close_ssk()
2446 * attempt the creation of the next subflow in mptcp_close_ssk()
2448 mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); in mptcp_close_ssk()
2450 __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); in mptcp_close_ssk()
2460 struct mptcp_subflow_context *subflow, *tmp; in __mptcp_close_subflow() local
2465 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in __mptcp_close_subflow()
2466 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_close_subflow()
2478 mptcp_close_ssk(sk, ssk, subflow); in __mptcp_close_subflow()
2495 struct mptcp_subflow_context *subflow, *tmp; in mptcp_check_fastclose() local
2503 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in mptcp_check_fastclose()
2504 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); in mptcp_check_fastclose()
2545 struct mptcp_subflow_context *subflow; in __mptcp_retrans() local
2577 mptcp_for_each_subflow(msk, subflow) { in __mptcp_retrans()
2578 if (READ_ONCE(subflow->scheduled)) { in __mptcp_retrans()
2581 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_retrans()
2583 ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_retrans()
2651 pr_debug("MP_FAIL doesn't respond, reset the subflow\n"); in mptcp_mp_fail_no_response()
2661 struct mptcp_subflow_context *subflow, *tmp; in mptcp_do_fastclose() local
2665 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_do_fastclose()
2666 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), in mptcp_do_fastclose()
2667 subflow, MPTCP_CF_FASTCLOSE); in mptcp_do_fastclose()
2834 pr_debug("Sending DATA_FIN on subflow %p\n", ssk); in mptcp_subflow_shutdown()
2897 struct mptcp_subflow_context *subflow; in mptcp_check_send_data_fin() local
2913 mptcp_for_each_subflow(msk, subflow) { in mptcp_check_send_data_fin()
2914 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); in mptcp_check_send_data_fin()
2990 struct mptcp_subflow_context *subflow; in __mptcp_close() local
3017 mptcp_for_each_subflow(msk, subflow) { in __mptcp_close()
3018 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_close()
3027 subflow->fail_tout = 0; in __mptcp_close()
3114 /* msk->subflow is still intact, the following will not free the first in mptcp_disconnect()
3115 * subflow in mptcp_disconnect()
3197 struct mptcp_subflow_context *subflow; in mptcp_sk_clone_init() local
3232 /* passive msk is created after the first/MPC subflow */ in mptcp_sk_clone_init()
3243 /* The msk maintain a ref to each subflow in the connections list */ in mptcp_sk_clone_init()
3245 subflow = mptcp_subflow_ctx(ssk); in mptcp_sk_clone_init()
3246 list_add(&subflow->node, &msk->conn_list); in mptcp_sk_clone_init()
3249 /* new mpc subflow takes ownership of the newly in mptcp_sk_clone_init()
3263 __mptcp_subflow_fully_established(msk, subflow, mp_opt); in mptcp_sk_clone_init()
3289 struct mptcp_subflow_context *subflow, *tmp; in mptcp_destroy_common() local
3295 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_destroy_common()
3296 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); in mptcp_destroy_common()
3312 /* allow the following to close even the initial subflow */ in mptcp_destroy()
3358 /* the following actions acquire the subflow socket lock in mptcp_release_cb()
3363 * the subflow socket lock in mptcp_release_cb()
3389 * On sk release avoid actions depending on the first subflow in mptcp_release_cb()
3400 /* MP_JOIN client subflow must wait for 4th ack before sending any data:
3401 * TCP can't schedule delack timer before the subflow is fully established.
3428 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_subflow_process_delegated() local
3429 struct sock *sk = subflow->conn; in mptcp_subflow_process_delegated()
3478 struct mptcp_subflow_context *subflow; in mptcp_finish_connect() local
3482 subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_connect()
3483 sk = subflow->conn; in mptcp_finish_connect()
3486 pr_debug("msk=%p, token=%u\n", sk, subflow->token); in mptcp_finish_connect()
3488 subflow->map_seq = subflow->iasn; in mptcp_finish_connect()
3489 subflow->map_subflow_seq = 1; in mptcp_finish_connect()
3491 /* the socket is not connected yet, no msk/subflow ops can access/race in mptcp_finish_connect()
3494 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
3510 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_join() local
3511 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join()
3515 pr_debug("msk=%p, subflow=%p\n", msk, subflow); in mptcp_finish_join()
3519 subflow->reset_reason = MPTCP_RST_EMPTCP; in mptcp_finish_join()
3523 /* active subflow, already present inside the conn_list */ in mptcp_finish_join()
3524 if (!list_empty(&subflow->node)) { in mptcp_finish_join()
3541 list_add_tail(&subflow->node, &msk->conn_list); in mptcp_finish_join()
3545 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
3552 subflow->reset_reason = MPTCP_RST_EPROHIBIT; in mptcp_finish_join()
3582 /* the first subflow is disconnected after close - see in mptcp_ioctl_outq()
3631 struct mptcp_subflow_context *subflow; in mptcp_connect() local
3641 subflow = mptcp_subflow_ctx(ssk); in mptcp_connect()
3647 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3649 if (subflow->request_mptcp) { in mptcp_connect()
3652 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3655 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3659 WRITE_ONCE(msk->write_seq, subflow->idsn); in mptcp_connect()
3660 WRITE_ONCE(msk->snd_nxt, subflow->idsn); in mptcp_connect()
3661 WRITE_ONCE(msk->snd_una, subflow->idsn); in mptcp_connect()
3666 * acquired the subflow socket lock, too. in mptcp_connect()
3816 * but no need to allocate the first subflow just to error out. in mptcp_stream_accept()
3827 pr_debug("newsk=%p, subflow is mptcp=%d\n", newsk, sk_is_mptcp(newsk)); in mptcp_stream_accept()
3829 struct mptcp_subflow_context *subflow; in mptcp_stream_accept() local
3832 subflow = mptcp_subflow_ctx(newsk); in mptcp_stream_accept()
3833 new_mptcp_sock = subflow->conn; in mptcp_stream_accept()
3835 /* is_mptcp should be false if subflow->conn is missing, see in mptcp_stream_accept()
3857 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
3858 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_stream_accept()
3864 /* Do late cleanup for the first subflow as necessary. Also in mptcp_stream_accept()
3989 struct mptcp_subflow_context *subflow; in mptcp_napi_poll() local
3993 while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { in mptcp_napi_poll()
3994 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_napi_poll()
3998 mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); in mptcp_napi_poll()
4002 * In both case must dequeue the subflow here - on the same in mptcp_napi_poll()
4006 clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); in mptcp_napi_poll()