70 	struct mptcp_subflow_context *subflow;  in __mptcp_socket_create()  local
81 	subflow = mptcp_subflow_ctx(ssock->sk);  in __mptcp_socket_create()
82 	list_add(&subflow->node, &msk->conn_list);  in __mptcp_socket_create()
84 	subflow->request_mptcp = 1;  in __mptcp_socket_create()
85 	subflow->subflow_id = msk->subflow_id++;  in __mptcp_socket_create()
87 	/* This is the first subflow, always with id 0 */  in __mptcp_socket_create()
88 	WRITE_ONCE(subflow->local_id, 0);  in __mptcp_socket_create()
95 /* If the MPC handshake is not started, returns the first subflow,
342 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skb()  local
352 	/* try to fetch required memory from subflow */  in __mptcp_move_skb()
362 	MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);  in __mptcp_move_skb()
480 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow)  in mptcp_timeout_from_subflow()  argument
482 	const struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_timeout_from_subflow()
484 	return inet_csk(ssk)->icsk_pending && !subflow->stale_count ?  in mptcp_timeout_from_subflow()
490 	struct mptcp_subflow_context *subflow;  in mptcp_set_timeout()  local
493 	mptcp_for_each_subflow(mptcp_sk(sk), subflow)  in mptcp_set_timeout()
494 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_set_timeout()
521 	struct mptcp_subflow_context *subflow;  in mptcp_send_ack()  local
523 	mptcp_for_each_subflow(msk, subflow)  in mptcp_send_ack()
524 		mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow));  in mptcp_send_ack()
553 	struct mptcp_subflow_context *subflow;  in mptcp_cleanup_rbuf()  local
561 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_cleanup_rbuf()
562 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_cleanup_rbuf()
578 	 * at the subflow level and the msk lock was not held, so this  in mptcp_check_data_fin()
623 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skbs_from_subflow()  local
651 		map_remaining = subflow->map_data_len -  in __mptcp_move_skbs_from_subflow()
652 				mptcp_subflow_get_map_offset(subflow);  in __mptcp_move_skbs_from_subflow()
671 			subflow->map_data_len = skb->len;  in __mptcp_move_skbs_from_subflow()
789 	struct mptcp_subflow_context *subflow;  in __mptcp_error_report()  local
792 	mptcp_for_each_subflow(msk, subflow)  in __mptcp_error_report()
793 		if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow)))  in __mptcp_error_report()
826 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_data_ready()  local
831 	 * subflow at msk destruction time, but we must avoid enqueuing  in mptcp_data_ready()
834 	if (unlikely(subflow->disposable))  in mptcp_data_ready()
885 	struct mptcp_subflow_context *tmp, *subflow;  in __mptcp_flush_join_list()  local
888 	list_for_each_entry_safe(subflow, tmp, join_list, node) {  in __mptcp_flush_join_list()
889 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_flush_join_list()
892 		list_move_tail(&subflow->node, &msk->conn_list);  in __mptcp_flush_join_list()
921 		/* each subflow already holds a reference to the sk, and the  in mptcp_schedule_work()
922 		 * workqueue is invoked by a subflow, so sk can't go away here.  in mptcp_schedule_work()
932 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_recv_lookup()  local
936 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_recv_lookup()
937 		if (READ_ONCE(subflow->data_avail))  in mptcp_subflow_recv_lookup()
938 			return mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_recv_lookup()
1062 	struct mptcp_subflow_context *subflow;  in mptcp_enter_memory_pressure()  local
1066 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_enter_memory_pressure()
1067 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_enter_memory_pressure()
1370 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_set_active()  argument
1372 	if (!subflow->stale)  in mptcp_subflow_set_active()
1375 	subflow->stale = 0;  in mptcp_subflow_set_active()
1376 	MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER);  in mptcp_subflow_set_active()
1379 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_active()  argument
1381 	if (unlikely(subflow->stale)) {  in mptcp_subflow_active()
1382 		u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp);  in mptcp_subflow_active()
1384 		if (subflow->stale_rcv_tstamp == rcv_tstamp)  in mptcp_subflow_active()
1387 		mptcp_subflow_set_active(subflow);  in mptcp_subflow_active()
1389 	return __mptcp_subflow_active(subflow);  in mptcp_subflow_active()
1397  * returns the subflow that will transmit the next DSS
1403 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_send()  local
1411 	/* pick the subflow with the lower wmem/wspace ratio */  in mptcp_subflow_get_send()
1417 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_send()
1418 		trace_mptcp_subflow_get_send(subflow);  in mptcp_subflow_get_send()
1419 		ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1420 		if (!mptcp_subflow_active(subflow))  in mptcp_subflow_get_send()
1423 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_subflow_get_send()
1424 		nr_active += !subflow->backup;  in mptcp_subflow_get_send()
1425 		pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1428 			subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate);  in mptcp_subflow_get_send()
1429 			pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1435 		if (linger_time < send_info[subflow->backup].linger_time) {  in mptcp_subflow_get_send()
1436 			send_info[subflow->backup].ssk = ssk;  in mptcp_subflow_get_send()
1437 			send_info[subflow->backup].linger_time = linger_time;  in mptcp_subflow_get_send()
1442 	/* pick the best backup if no other subflow is active */  in mptcp_subflow_get_send()
1452 	 *   otherwise do not use the picked, slower, subflow  in mptcp_subflow_get_send()
1453 	 * We select the subflow with the shorter estimated time to flush  in mptcp_subflow_get_send()
1455 	 * to check that subflow has a non empty cwin.  in mptcp_subflow_get_send()
1466 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_get_send()
1467 	subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +  in mptcp_subflow_get_send()
1498 	 * that has been handed to the subflow for transmission  in mptcp_update_post_push()
1569 		struct mptcp_subflow_context *subflow;  in __mptcp_push_pending()  local
1577 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_push_pending()
1578 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_push_pending()
1579 				mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_push_pending()
1582 				ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_push_pending()
1590 					/* Need to lock the new subflow only if different  in __mptcp_push_pending()
1612 	/* at this point we held the socket lock for the last subflow we used */  in __mptcp_push_pending()
1635 		struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_subflow_push_pending()  local
1638 		/* check for a different subflow usage only after  in __mptcp_subflow_push_pending()
1642 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1654 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1655 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1662 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_subflow_push_pending()
1663 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1664 				xmit_ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_subflow_push_pending()
1666 					mptcp_subflow_delegate(subflow,  in __mptcp_subflow_push_pending()
1710 	 * first subflow right now. Otherwise we are in the defer_connect  in mptcp_sendmsg_fastopen()
1711 	 * path, and the first subflow must be already present.  in mptcp_sendmsg_fastopen()
1713 	 * fastopen attempt, no need to check for additional subflow status.  in mptcp_sendmsg_fastopen()
1956 	struct mptcp_subflow_context *subflow;  in mptcp_rcv_space_adjust()  local
1980 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
1985 		tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));  in mptcp_rcv_space_adjust()
2025 			 * get drops at subflow level if skbs can't be moved to  in mptcp_rcv_space_adjust()
2029 			mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
2033 				ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_rcv_space_adjust()
2275 /* Find an idle subflow.  Return NULL if there is unacked data at tcp
2278  * A backup subflow is returned only if that is the only kind available.
2283 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_retrans()  local
2286 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_retrans()
2287 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_retrans()
2289 		if (!__mptcp_subflow_active(subflow))  in mptcp_subflow_get_retrans()
2295 			min_stale_count = min_t(int, min_stale_count, subflow->stale_count);  in mptcp_subflow_get_retrans()
2299 		if (subflow->backup) {  in mptcp_subflow_get_retrans()
2358  * clean completely the subflow status when the subflow reaches
2362 				       struct mptcp_subflow_context *subflow,  in __mptcp_subflow_disconnect()  argument
2367 		/* The MPTCP code never wait on the subflow sockets, TCP-level  in __mptcp_subflow_disconnect()
2371 		mptcp_subflow_ctx_reset(subflow);  in __mptcp_subflow_disconnect()
2377 /* subflow sockets can be either outgoing (connect) or incoming
2386 			      struct mptcp_subflow_context *subflow,  in __mptcp_close_ssk()  argument
2392 	/* If the first subflow moved to a close state before accept, e.g. due  in __mptcp_close_ssk()
2393 	 * to an incoming reset or listener shutdown, the subflow socket is  in __mptcp_close_ssk()
2409 		list_del(&subflow->node);  in __mptcp_close_ssk()
2419 		subflow->send_fastclose = 1;  in __mptcp_close_ssk()
2424 		__mptcp_subflow_disconnect(ssk, subflow, flags);  in __mptcp_close_ssk()
2430 	subflow->disposable = 1;  in __mptcp_close_ssk()
2438 		kfree_rcu(subflow, rcu);  in __mptcp_close_ssk()
2440 		/* otherwise tcp will dispose of the ssk and subflow ctx */  in __mptcp_close_ssk()
2479 		     struct mptcp_subflow_context *subflow)  in mptcp_close_ssk()  argument
2484 	/* subflow aborted before reaching the fully_established status  in mptcp_close_ssk()
2485 	 * attempt the creation of the next subflow  in mptcp_close_ssk()
2487 	mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow);  in mptcp_close_ssk()
2489 	__mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);  in mptcp_close_ssk()
2499 	struct mptcp_subflow_context *subflow, *tmp;  in __mptcp_close_subflow()  local
2504 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in __mptcp_close_subflow()
2505 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close_subflow()
2514 		mptcp_close_ssk(sk, ssk, subflow);  in __mptcp_close_subflow()
2531 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_check_fastclose()  local
2539 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in mptcp_check_fastclose()
2540 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_fastclose()
2581 	struct mptcp_subflow_context *subflow;  in __mptcp_retrans()  local
2613 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_retrans()
2614 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_retrans()
2617 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_retrans()
2619 			ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_retrans()
2687 	pr_debug("MP_FAIL doesn't respond, reset the subflow");  in mptcp_mp_fail_no_response()
2697 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_do_fastclose()  local
2701 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_do_fastclose()
2702 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),  in mptcp_do_fastclose()
2703 				  subflow, MPTCP_CF_FASTCLOSE);  in mptcp_do_fastclose()
2867 			pr_debug("Sending DATA_FIN on subflow %p", ssk);  in mptcp_subflow_shutdown()
2925 	struct mptcp_subflow_context *subflow;  in mptcp_check_send_data_fin()  local
2941 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_check_send_data_fin()
2942 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_send_data_fin()
3020 	struct mptcp_subflow_context *subflow;  in __mptcp_close()  local
3047 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_close()
3048 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close()
3057 			subflow->fail_tout = 0;  in __mptcp_close()
3146 	/* msk->subflow is still intact, the following will not free the first  in mptcp_disconnect()
3147 	 * subflow  in mptcp_disconnect()
3232 	struct mptcp_subflow_context *subflow;  in mptcp_sk_clone_init()  local
3267 	/* passive msk is created after the first/MPC subflow */  in mptcp_sk_clone_init()
3278 	/* The msk maintain a ref to each subflow in the connections list */  in mptcp_sk_clone_init()
3280 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_sk_clone_init()
3281 	list_add(&subflow->node, &msk->conn_list);  in mptcp_sk_clone_init()
3284 	/* new mpc subflow takes ownership of the newly  in mptcp_sk_clone_init()
3298 		__mptcp_subflow_fully_established(msk, subflow, mp_opt);  in mptcp_sk_clone_init()
3324 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_destroy_common()  local
3330 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_destroy_common()
3331 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);  in mptcp_destroy_common()
3354 	/* allow the following to close even the initial subflow */  in mptcp_destroy()
3402 		/* the following actions acquire the subflow socket lock  in mptcp_release_cb()
3407 		 *    the subflow socket lock  in mptcp_release_cb()
3428 		 * On sk release avoid actions depending on the first subflow  in mptcp_release_cb()
3441 /* MP_JOIN client subflow must wait for 4th ack before sending any data:
3442  * TCP can't schedule delack timer before the subflow is fully established.
3469 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_process_delegated()  local
3470 	struct sock *sk = subflow->conn;  in mptcp_subflow_process_delegated()
3519 	struct mptcp_subflow_context *subflow;  in mptcp_finish_connect()  local
3523 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_connect()
3524 	sk = subflow->conn;  in mptcp_finish_connect()
3527 	pr_debug("msk=%p, token=%u", sk, subflow->token);  in mptcp_finish_connect()
3529 	subflow->map_seq = subflow->iasn;  in mptcp_finish_connect()
3530 	subflow->map_subflow_seq = 1;  in mptcp_finish_connect()
3532 	/* the socket is not connected yet, no msk/subflow ops can access/race  in mptcp_finish_connect()
3535 	WRITE_ONCE(msk->local_key, subflow->local_key);  in mptcp_finish_connect()
3551 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_join()  local
3552 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);  in mptcp_finish_join()
3556 	pr_debug("msk=%p, subflow=%p", msk, subflow);  in mptcp_finish_join()
3560 		subflow->reset_reason = MPTCP_RST_EMPTCP;  in mptcp_finish_join()
3564 	/* active subflow, already present inside the conn_list */  in mptcp_finish_join()
3565 	if (!list_empty(&subflow->node)) {  in mptcp_finish_join()
3582 			list_add_tail(&subflow->node, &msk->conn_list);  in mptcp_finish_join()
3586 		list_add_tail(&subflow->node, &msk->join_list);  in mptcp_finish_join()
3593 		subflow->reset_reason = MPTCP_RST_EPROHIBIT;  in mptcp_finish_join()
3629 		/* the first subflow is disconnected after close - see  in mptcp_ioctl_outq()
3676 					 struct mptcp_subflow_context *subflow)  in mptcp_subflow_early_fallback()  argument
3678 	subflow->request_mptcp = 0;  in mptcp_subflow_early_fallback()
3684 	struct mptcp_subflow_context *subflow;  in mptcp_connect()  local
3694 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_connect()
3700 		mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3702 	if (subflow->request_mptcp && mptcp_token_new_connect(ssk)) {  in mptcp_connect()
3704 		mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3710 	 * acquired the subflow socket lock, too.  in mptcp_connect()
3861 	 * but no need to allocate the first subflow just to error out.  in mptcp_stream_accept()
3872 	pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk));  in mptcp_stream_accept()
3874 		struct mptcp_subflow_context *subflow;  in mptcp_stream_accept()  local
3877 		subflow = mptcp_subflow_ctx(newsk);  in mptcp_stream_accept()
3878 		new_mptcp_sock = subflow->conn;  in mptcp_stream_accept()
3880 		/* is_mptcp should be false if subflow->conn is missing, see  in mptcp_stream_accept()
3902 		mptcp_for_each_subflow(msk, subflow) {  in mptcp_stream_accept()
3903 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_stream_accept()
3909 		/* Do late cleanup for the first subflow as necessary. Also  in mptcp_stream_accept()
4036 	struct mptcp_subflow_context *subflow;  in mptcp_napi_poll()  local
4040 	while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) {  in mptcp_napi_poll()
4041 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_napi_poll()
4045 			mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0));  in mptcp_napi_poll()
4049 			 * In both case must dequeue the subflow here - on the same  in mptcp_napi_poll()
4053 			clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status);  in mptcp_napi_poll()