Lines Matching full:subflow

70 	struct mptcp_subflow_context *subflow;  in __mptcp_socket_create()  local
81 subflow = mptcp_subflow_ctx(ssock->sk); in __mptcp_socket_create()
82 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
84 subflow->request_mptcp = 1; in __mptcp_socket_create()
85 subflow->subflow_id = msk->subflow_id++; in __mptcp_socket_create()
87 /* This is the first subflow, always with id 0 */ in __mptcp_socket_create()
88 WRITE_ONCE(subflow->local_id, 0); in __mptcp_socket_create()
95 /* If the MPC handshake is not started, returns the first subflow,
342 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skb() local
352 /* try to fetch required memory from subflow */ in __mptcp_move_skb()
362 MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); in __mptcp_move_skb()
480 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) in mptcp_timeout_from_subflow() argument
482 const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_timeout_from_subflow()
484 return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? in mptcp_timeout_from_subflow()
490 struct mptcp_subflow_context *subflow; in mptcp_set_timeout() local
493 mptcp_for_each_subflow(mptcp_sk(sk), subflow) in mptcp_set_timeout()
494 tout = max(tout, mptcp_timeout_from_subflow(subflow)); in mptcp_set_timeout()
521 struct mptcp_subflow_context *subflow; in mptcp_send_ack() local
523 mptcp_for_each_subflow(msk, subflow) in mptcp_send_ack()
524 mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); in mptcp_send_ack()
553 struct mptcp_subflow_context *subflow; in mptcp_cleanup_rbuf() local
561 mptcp_for_each_subflow(msk, subflow) { in mptcp_cleanup_rbuf()
562 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_cleanup_rbuf()
578 * at the subflow level and the msk lock was not held, so this in mptcp_check_data_fin()
623 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_move_skbs_from_subflow() local
651 map_remaining = subflow->map_data_len - in __mptcp_move_skbs_from_subflow()
652 mptcp_subflow_get_map_offset(subflow); in __mptcp_move_skbs_from_subflow()
671 subflow->map_data_len = skb->len; in __mptcp_move_skbs_from_subflow()
789 struct mptcp_subflow_context *subflow; in __mptcp_error_report() local
792 mptcp_for_each_subflow(msk, subflow) in __mptcp_error_report()
793 if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) in __mptcp_error_report()
826 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_data_ready() local
831 * subflow at msk destruction time, but we must avoid enqueuing in mptcp_data_ready()
834 if (unlikely(subflow->disposable)) in mptcp_data_ready()
885 struct mptcp_subflow_context *tmp, *subflow; in __mptcp_flush_join_list() local
888 list_for_each_entry_safe(subflow, tmp, join_list, node) { in __mptcp_flush_join_list()
889 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_flush_join_list()
892 list_move_tail(&subflow->node, &msk->conn_list); in __mptcp_flush_join_list()
921 /* each subflow already holds a reference to the sk, and the in mptcp_schedule_work()
922 * workqueue is invoked by a subflow, so sk can't go away here. in mptcp_schedule_work()
932 struct mptcp_subflow_context *subflow; in mptcp_subflow_recv_lookup() local
936 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
937 if (READ_ONCE(subflow->data_avail)) in mptcp_subflow_recv_lookup()
938 return mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_recv_lookup()
1062 struct mptcp_subflow_context *subflow; in mptcp_enter_memory_pressure() local
1066 mptcp_for_each_subflow(msk, subflow) { in mptcp_enter_memory_pressure()
1067 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_enter_memory_pressure()
1370 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) in mptcp_subflow_set_active() argument
1372 if (!subflow->stale) in mptcp_subflow_set_active()
1375 subflow->stale = 0; in mptcp_subflow_set_active()
1376 MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); in mptcp_subflow_set_active()
1379 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) in mptcp_subflow_active() argument
1381 if (unlikely(subflow->stale)) { in mptcp_subflow_active()
1382 u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); in mptcp_subflow_active()
1384 if (subflow->stale_rcv_tstamp == rcv_tstamp) in mptcp_subflow_active()
1387 mptcp_subflow_set_active(subflow); in mptcp_subflow_active()
1389 return __mptcp_subflow_active(subflow); in mptcp_subflow_active()
1397 * returns the subflow that will transmit the next DSS
1403 struct mptcp_subflow_context *subflow; in mptcp_subflow_get_send() local
1411 /* pick the subflow with the lower wmem/wspace ratio */ in mptcp_subflow_get_send()
1417 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1418 trace_mptcp_subflow_get_send(subflow); in mptcp_subflow_get_send()
1419 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_send()
1420 if (!mptcp_subflow_active(subflow)) in mptcp_subflow_get_send()
1423 tout = max(tout, mptcp_timeout_from_subflow(subflow)); in mptcp_subflow_get_send()
1424 nr_active += !subflow->backup; in mptcp_subflow_get_send()
1425 pace = subflow->avg_pacing_rate; in mptcp_subflow_get_send()
1428 subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); in mptcp_subflow_get_send()
1429 pace = subflow->avg_pacing_rate; in mptcp_subflow_get_send()
1435 if (linger_time < send_info[subflow->backup].linger_time) { in mptcp_subflow_get_send()
1436 send_info[subflow->backup].ssk = ssk; in mptcp_subflow_get_send()
1437 send_info[subflow->backup].linger_time = linger_time; in mptcp_subflow_get_send()
1442 /* pick the best backup if no other subflow is active */ in mptcp_subflow_get_send()
1452 * otherwise do not use the picked, slower, subflow in mptcp_subflow_get_send()
1453 * We select the subflow with the shorter estimated time to flush in mptcp_subflow_get_send()
1455 * to check that subflow has a non empty cwin. in mptcp_subflow_get_send()
1466 subflow = mptcp_subflow_ctx(ssk); in mptcp_subflow_get_send()
1467 subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + in mptcp_subflow_get_send()
1498 * that has been handed to the subflow for transmission in mptcp_update_post_push()
1569 struct mptcp_subflow_context *subflow; in __mptcp_push_pending() local
1577 mptcp_for_each_subflow(msk, subflow) { in __mptcp_push_pending()
1578 if (READ_ONCE(subflow->scheduled)) { in __mptcp_push_pending()
1579 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_push_pending()
1582 ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_push_pending()
1590 /* Need to lock the new subflow only if different in __mptcp_push_pending()
1612 /* at this point we held the socket lock for the last subflow we used */ in __mptcp_push_pending()
1635 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in __mptcp_subflow_push_pending() local
1638 /* check for a different subflow usage only after in __mptcp_subflow_push_pending()
1642 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_subflow_push_pending()
1654 if (READ_ONCE(subflow->scheduled)) { in __mptcp_subflow_push_pending()
1655 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_subflow_push_pending()
1662 mptcp_for_each_subflow(msk, subflow) { in __mptcp_subflow_push_pending()
1663 if (READ_ONCE(subflow->scheduled)) { in __mptcp_subflow_push_pending()
1664 xmit_ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_subflow_push_pending()
1666 mptcp_subflow_delegate(subflow, in __mptcp_subflow_push_pending()
1710 * first subflow right now. Otherwise we are in the defer_connect in mptcp_sendmsg_fastopen()
1711 * path, and the first subflow must be already present. in mptcp_sendmsg_fastopen()
1713 * fastopen attempt, no need to check for additional subflow status. in mptcp_sendmsg_fastopen()
1956 struct mptcp_subflow_context *subflow; in mptcp_rcv_space_adjust() local
1980 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1985 tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); in mptcp_rcv_space_adjust()
2025 * get drops at subflow level if skbs can't be moved to in mptcp_rcv_space_adjust()
2029 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
2033 ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_rcv_space_adjust()
2275 /* Find an idle subflow. Return NULL if there is unacked data at tcp
2278 * A backup subflow is returned only if that is the only kind available.
2283 struct mptcp_subflow_context *subflow; in mptcp_subflow_get_retrans() local
2286 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
2287 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_subflow_get_retrans()
2289 if (!__mptcp_subflow_active(subflow)) in mptcp_subflow_get_retrans()
2295 min_stale_count = min_t(int, min_stale_count, subflow->stale_count); in mptcp_subflow_get_retrans()
2299 if (subflow->backup) { in mptcp_subflow_get_retrans()
2358 * clean completely the subflow status when the subflow reaches
2362 struct mptcp_subflow_context *subflow, in __mptcp_subflow_disconnect() argument
2367 /* The MPTCP code never wait on the subflow sockets, TCP-level in __mptcp_subflow_disconnect()
2371 mptcp_subflow_ctx_reset(subflow); in __mptcp_subflow_disconnect()
2377 /* subflow sockets can be either outgoing (connect) or incoming
2386 struct mptcp_subflow_context *subflow, in __mptcp_close_ssk() argument
2392 /* If the first subflow moved to a close state before accept, e.g. due in __mptcp_close_ssk()
2393 * to an incoming reset or listener shutdown, the subflow socket is in __mptcp_close_ssk()
2409 list_del(&subflow->node); in __mptcp_close_ssk()
2419 subflow->send_fastclose = 1; in __mptcp_close_ssk()
2424 __mptcp_subflow_disconnect(ssk, subflow, flags); in __mptcp_close_ssk()
2430 subflow->disposable = 1; in __mptcp_close_ssk()
2438 kfree_rcu(subflow, rcu); in __mptcp_close_ssk()
2440 /* otherwise tcp will dispose of the ssk and subflow ctx */ in __mptcp_close_ssk()
2479 struct mptcp_subflow_context *subflow) in mptcp_close_ssk() argument
2484 /* subflow aborted before reaching the fully_established status in mptcp_close_ssk()
2485 * attempt the creation of the next subflow in mptcp_close_ssk()
2487 mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); in mptcp_close_ssk()
2489 __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); in mptcp_close_ssk()
2499 struct mptcp_subflow_context *subflow, *tmp; in __mptcp_close_subflow() local
2504 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in __mptcp_close_subflow()
2505 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_close_subflow()
2514 mptcp_close_ssk(sk, ssk, subflow); in __mptcp_close_subflow()
2531 struct mptcp_subflow_context *subflow, *tmp; in mptcp_check_fastclose() local
2539 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in mptcp_check_fastclose()
2540 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); in mptcp_check_fastclose()
2581 struct mptcp_subflow_context *subflow; in __mptcp_retrans() local
2613 mptcp_for_each_subflow(msk, subflow) { in __mptcp_retrans()
2614 if (READ_ONCE(subflow->scheduled)) { in __mptcp_retrans()
2617 mptcp_subflow_set_scheduled(subflow, false); in __mptcp_retrans()
2619 ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_retrans()
2687 pr_debug("MP_FAIL doesn't respond, reset the subflow"); in mptcp_mp_fail_no_response()
2697 struct mptcp_subflow_context *subflow, *tmp; in mptcp_do_fastclose() local
2701 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_do_fastclose()
2702 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), in mptcp_do_fastclose()
2703 subflow, MPTCP_CF_FASTCLOSE); in mptcp_do_fastclose()
2867 pr_debug("Sending DATA_FIN on subflow %p", ssk); in mptcp_subflow_shutdown()
2925 struct mptcp_subflow_context *subflow; in mptcp_check_send_data_fin() local
2941 mptcp_for_each_subflow(msk, subflow) { in mptcp_check_send_data_fin()
2942 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); in mptcp_check_send_data_fin()
3020 struct mptcp_subflow_context *subflow; in __mptcp_close() local
3047 mptcp_for_each_subflow(msk, subflow) { in __mptcp_close()
3048 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in __mptcp_close()
3057 subflow->fail_tout = 0; in __mptcp_close()
3146 /* msk->subflow is still intact, the following will not free the first in mptcp_disconnect()
3147 * subflow in mptcp_disconnect()
3232 struct mptcp_subflow_context *subflow; in mptcp_sk_clone_init() local
3267 /* passive msk is created after the first/MPC subflow */ in mptcp_sk_clone_init()
3278 /* The msk maintain a ref to each subflow in the connections list */ in mptcp_sk_clone_init()
3280 subflow = mptcp_subflow_ctx(ssk); in mptcp_sk_clone_init()
3281 list_add(&subflow->node, &msk->conn_list); in mptcp_sk_clone_init()
3284 /* new mpc subflow takes ownership of the newly in mptcp_sk_clone_init()
3298 __mptcp_subflow_fully_established(msk, subflow, mp_opt); in mptcp_sk_clone_init()
3324 struct mptcp_subflow_context *subflow, *tmp; in mptcp_destroy_common() local
3330 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_destroy_common()
3331 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); in mptcp_destroy_common()
3354 /* allow the following to close even the initial subflow */ in mptcp_destroy()
3402 /* the following actions acquire the subflow socket lock in mptcp_release_cb()
3407 * the subflow socket lock in mptcp_release_cb()
3428 * On sk release avoid actions depending on the first subflow in mptcp_release_cb()
3441 /* MP_JOIN client subflow must wait for 4th ack before sending any data:
3442 * TCP can't schedule delack timer before the subflow is fully established.
3469 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_subflow_process_delegated() local
3470 struct sock *sk = subflow->conn; in mptcp_subflow_process_delegated()
3519 struct mptcp_subflow_context *subflow; in mptcp_finish_connect() local
3523 subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_connect()
3524 sk = subflow->conn; in mptcp_finish_connect()
3527 pr_debug("msk=%p, token=%u", sk, subflow->token); in mptcp_finish_connect()
3529 subflow->map_seq = subflow->iasn; in mptcp_finish_connect()
3530 subflow->map_subflow_seq = 1; in mptcp_finish_connect()
3532 /* the socket is not connected yet, no msk/subflow ops can access/race in mptcp_finish_connect()
3535 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
3551 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); in mptcp_finish_join() local
3552 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join()
3556 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
3560 subflow->reset_reason = MPTCP_RST_EMPTCP; in mptcp_finish_join()
3564 /* active subflow, already present inside the conn_list */ in mptcp_finish_join()
3565 if (!list_empty(&subflow->node)) { in mptcp_finish_join()
3582 list_add_tail(&subflow->node, &msk->conn_list); in mptcp_finish_join()
3586 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
3593 subflow->reset_reason = MPTCP_RST_EPROHIBIT; in mptcp_finish_join()
3629 /* the first subflow is disconnected after close - see in mptcp_ioctl_outq()
3676 struct mptcp_subflow_context *subflow) in mptcp_subflow_early_fallback() argument
3678 subflow->request_mptcp = 0; in mptcp_subflow_early_fallback()
3684 struct mptcp_subflow_context *subflow; in mptcp_connect() local
3694 subflow = mptcp_subflow_ctx(ssk); in mptcp_connect()
3700 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3702 if (subflow->request_mptcp && mptcp_token_new_connect(ssk)) { in mptcp_connect()
3704 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3710 * acquired the subflow socket lock, too. in mptcp_connect()
3861 * but no need to allocate the first subflow just to error out. in mptcp_stream_accept()
3872 pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk)); in mptcp_stream_accept()
3874 struct mptcp_subflow_context *subflow; in mptcp_stream_accept() local
3877 subflow = mptcp_subflow_ctx(newsk); in mptcp_stream_accept()
3878 new_mptcp_sock = subflow->conn; in mptcp_stream_accept()
3880 /* is_mptcp should be false if subflow->conn is missing, see in mptcp_stream_accept()
3902 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
3903 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_stream_accept()
3909 /* Do late cleanup for the first subflow as necessary. Also in mptcp_stream_accept()
4036 struct mptcp_subflow_context *subflow; in mptcp_napi_poll() local
4040 while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { in mptcp_napi_poll()
4041 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); in mptcp_napi_poll()
4045 mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); in mptcp_napi_poll()
4049 * In both case must dequeue the subflow here - on the same in mptcp_napi_poll()
4053 clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); in mptcp_napi_poll()