23 static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)  in rxrpc_check_tx_space()  argument
26 		min_t(unsigned int, call->tx_winsize,  in rxrpc_check_tx_space()
27 		      call->cong_cwnd + call->cong_extra);  in rxrpc_check_tx_space()
28 	rxrpc_seq_t tx_win = READ_ONCE(call->tx_hard_ack);  in rxrpc_check_tx_space()
32 	return call->tx_top - tx_win < win_size;  in rxrpc_check_tx_space()
39 					 struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_intr()  argument
44 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_intr()
47 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_intr()
48 			return call->error;  in rxrpc_wait_for_tx_window_intr()
53 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_intr()
54 		mutex_unlock(&call->user_mutex);  in rxrpc_wait_for_tx_window_intr()
56 		if (mutex_lock_interruptible(&call->user_mutex) < 0)  in rxrpc_wait_for_tx_window_intr()
66 					    struct rxrpc_call *call)  in rxrpc_wait_for_tx_window_waitall()  argument
71 	rtt = READ_ONCE(call->peer->srtt_us) >> 3;  in rxrpc_wait_for_tx_window_waitall()
77 	tx_start = READ_ONCE(call->tx_hard_ack);  in rxrpc_wait_for_tx_window_waitall()
82 		tx_win = READ_ONCE(call->tx_hard_ack);  in rxrpc_wait_for_tx_window_waitall()
83 		if (rxrpc_check_tx_space(call, &tx_win))  in rxrpc_wait_for_tx_window_waitall()
86 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_waitall()
87 			return call->error;  in rxrpc_wait_for_tx_window_waitall()
98 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_waitall()
107 					    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_nonintr()  argument
112 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_nonintr()
115 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_nonintr()
116 			return call->error;  in rxrpc_wait_for_tx_window_nonintr()
118 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_nonintr()
128 				    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window()  argument
136 	       call->tx_hard_ack, call->tx_top, call->tx_winsize);  in rxrpc_wait_for_tx_window()
138 	add_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
140 	switch (call->interruptibility) {  in rxrpc_wait_for_tx_window()
143 			ret = rxrpc_wait_for_tx_window_waitall(rx, call);  in rxrpc_wait_for_tx_window()
145 			ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
150 		ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
154 	remove_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
163 static inline void rxrpc_instant_resend(struct rxrpc_call *call, int ix)  in rxrpc_instant_resend()  argument
165 	spin_lock_bh(&call->lock);  in rxrpc_instant_resend()
167 	if (call->state < RXRPC_CALL_COMPLETE) {  in rxrpc_instant_resend()
168 		call->rxtx_annotations[ix] =  in rxrpc_instant_resend()
169 			(call->rxtx_annotations[ix] & RXRPC_TX_ANNO_LAST) |  in rxrpc_instant_resend()
171 		if (!test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))  in rxrpc_instant_resend()
172 			rxrpc_queue_call(call);  in rxrpc_instant_resend()
175 	spin_unlock_bh(&call->lock);  in rxrpc_instant_resend()
179  * Notify the owner of the call that the transmit phase is ended and the last
182 static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_notify_end_tx()  argument
186 		notify_end_tx(&rx->sk, call, call->user_call_ID);  in rxrpc_notify_end_tx()
194 static int rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_queue_packet()  argument
206 	ASSERTCMP(seq, ==, call->tx_top + 1);  in rxrpc_queue_packet()
218 	call->rxtx_annotations[ix] = annotation;  in rxrpc_queue_packet()
220 	call->rxtx_buffer[ix] = skb;  in rxrpc_queue_packet()
221 	call->tx_top = seq;  in rxrpc_queue_packet()
223 		trace_rxrpc_transmit(call, rxrpc_transmit_queue_last);  in rxrpc_queue_packet()
225 		trace_rxrpc_transmit(call, rxrpc_transmit_queue);  in rxrpc_queue_packet()
227 	if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {  in rxrpc_queue_packet()
229 		write_lock_bh(&call->state_lock);  in rxrpc_queue_packet()
230 		switch (call->state) {  in rxrpc_queue_packet()
232 			call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;  in rxrpc_queue_packet()
233 			rxrpc_notify_end_tx(rx, call, notify_end_tx);  in rxrpc_queue_packet()
236 			call->state = RXRPC_CALL_SERVER_SEND_REPLY;  in rxrpc_queue_packet()
238 			WRITE_ONCE(call->ack_at, now + MAX_JIFFY_OFFSET);  in rxrpc_queue_packet()
239 			if (call->ackr_reason == RXRPC_ACK_DELAY)  in rxrpc_queue_packet()
240 				call->ackr_reason = 0;  in rxrpc_queue_packet()
241 			trace_rxrpc_timer(call, rxrpc_timer_init_for_send_reply, now);  in rxrpc_queue_packet()
246 			call->state = RXRPC_CALL_SERVER_AWAIT_ACK;  in rxrpc_queue_packet()
247 			rxrpc_notify_end_tx(rx, call, notify_end_tx);  in rxrpc_queue_packet()
252 		write_unlock_bh(&call->state_lock);  in rxrpc_queue_packet()
255 	if (seq == 1 && rxrpc_is_client_call(call))  in rxrpc_queue_packet()
256 		rxrpc_expose_client_call(call);  in rxrpc_queue_packet()
258 	ret = rxrpc_send_data_packet(call, skb, false);  in rxrpc_queue_packet()
264 			rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR,  in rxrpc_queue_packet()
269 		rxrpc_instant_resend(call, ix);  in rxrpc_queue_packet()
272 		unsigned long resend_at = now + call->peer->rto_j;  in rxrpc_queue_packet()
274 		WRITE_ONCE(call->resend_at, resend_at);  in rxrpc_queue_packet()
275 		rxrpc_reduce_call_timer(call, resend_at, now,  in rxrpc_queue_packet()
288  * - The caller holds the call user access mutex, but not the socket lock.
291 			   struct rxrpc_call *call,  in rxrpc_send_data()  argument
312 	if (call->tx_total_len != -1) {  in rxrpc_send_data()
313 		if (len > call->tx_total_len)  in rxrpc_send_data()
315 		if (!more && len != call->tx_total_len)  in rxrpc_send_data()
319 	skb = call->tx_pending;  in rxrpc_send_data()
320 	call->tx_pending = NULL;  in rxrpc_send_data()
326 		if (call->ackr_reason == RXRPC_ACK_PING_RESPONSE)  in rxrpc_send_data()
327 			rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_send_data()
334 			if (!rxrpc_check_tx_space(call, NULL)) {  in rxrpc_send_data()
338 				ret = rxrpc_wait_for_tx_window(rx, call,  in rxrpc_send_data()
346 			max -= call->conn->security_size;  in rxrpc_send_data()
347 			max &= ~(call->conn->size_align - 1UL);  in rxrpc_send_data()
353 			space = chunk + call->conn->size_align;  in rxrpc_send_data()
354 			space &= ~(call->conn->size_align - 1UL);  in rxrpc_send_data()
356 			size = space + call->conn->security_size;  in rxrpc_send_data()
374 			_debug("HS: %u", call->conn->security_size);  in rxrpc_send_data()
375 			skb_reserve(skb, call->conn->security_size);  in rxrpc_send_data()
376 			skb->len += call->conn->security_size;  in rxrpc_send_data()
411 			if (call->tx_total_len != -1)  in rxrpc_send_data()
412 				call->tx_total_len -= copy;  in rxrpc_send_data()
415 		/* check for the far side aborting the call or a network error  in rxrpc_send_data()
417 		if (call->state == RXRPC_CALL_COMPLETE)  in rxrpc_send_data()
423 			struct rxrpc_connection *conn = call->conn;  in rxrpc_send_data()
437 			seq = call->tx_top + 1;  in rxrpc_send_data()
445 			else if (call->tx_top - call->tx_hard_ack <  in rxrpc_send_data()
446 				 call->tx_winsize)  in rxrpc_send_data()
449 			ret = call->security->secure_packet(  in rxrpc_send_data()
450 				call, skb, skb->mark, skb->head);  in rxrpc_send_data()
454 			ret = rxrpc_queue_packet(rx, call, skb,  in rxrpc_send_data()
465 	call->tx_pending = skb;  in rxrpc_send_data()
471 	_leave(" = %d", call->error);  in rxrpc_send_data()
472 	return call->error;  in rxrpc_send_data()
512 				p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
516 				p->call.user_call_ID = *(unsigned long *)  in rxrpc_sendmsg_cmsg()
554 			if (p->call.tx_total_len != -1 || len != sizeof(__s64))  in rxrpc_sendmsg_cmsg()
556 			p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
557 			if (p->call.tx_total_len < 0)  in rxrpc_sendmsg_cmsg()
564 			memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);  in rxrpc_sendmsg_cmsg()
565 			p->call.nr_timeouts = len / 4;  in rxrpc_sendmsg_cmsg()
566 			if (p->call.timeouts.hard > INT_MAX / HZ)  in rxrpc_sendmsg_cmsg()
568 			if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
570 			if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
581 	if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)  in rxrpc_sendmsg_cmsg()
588  * Create a new client call for sendmsg().
590  * - If it returns a call, the call's lock will need releasing by the caller.
596 	__acquires(&call->user_mutex)  in rxrpc_new_client_call_for_sendmsg()
599 	struct rxrpc_call *call;  in rxrpc_new_client_call_for_sendmsg()  local
622 	call = rxrpc_new_client_call(rx, &cp, srx, &p->call, GFP_KERNEL,  in rxrpc_new_client_call_for_sendmsg()
627 	_leave(" = %p\n", call);  in rxrpc_new_client_call_for_sendmsg()
628 	return call;  in rxrpc_new_client_call_for_sendmsg()
632  * send a message forming part of a client call through an RxRPC socket
638 	__releases(&call->user_mutex)  in rxrpc_do_sendmsg()
641 	struct rxrpc_call *call;  in rxrpc_do_sendmsg()  local
646 		.call.tx_total_len	= -1,  in rxrpc_do_sendmsg()
647 		.call.user_call_ID	= 0,  in rxrpc_do_sendmsg()
648 		.call.nr_timeouts	= 0,  in rxrpc_do_sendmsg()
649 		.call.interruptibility	= RXRPC_INTERRUPTIBLE,  in rxrpc_do_sendmsg()
666 		ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
670 	call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
671 	if (!call) {  in rxrpc_do_sendmsg()
675 		call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);  in rxrpc_do_sendmsg()
677 		if (IS_ERR(call))  in rxrpc_do_sendmsg()
678 			return PTR_ERR(call);  in rxrpc_do_sendmsg()
679 		/* ... and we have the call lock. */  in rxrpc_do_sendmsg()
681 		if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE)  in rxrpc_do_sendmsg()
684 		switch (READ_ONCE(call->state)) {  in rxrpc_do_sendmsg()
689 			rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_do_sendmsg()
696 		ret = mutex_lock_interruptible(&call->user_mutex);  in rxrpc_do_sendmsg()
703 		if (p.call.tx_total_len != -1) {  in rxrpc_do_sendmsg()
705 			if (call->tx_total_len != -1 ||  in rxrpc_do_sendmsg()
706 			    call->tx_pending ||  in rxrpc_do_sendmsg()
707 			    call->tx_top != 0)  in rxrpc_do_sendmsg()
709 			call->tx_total_len = p.call.tx_total_len;  in rxrpc_do_sendmsg()
713 	switch (p.call.nr_timeouts) {  in rxrpc_do_sendmsg()
715 		j = msecs_to_jiffies(p.call.timeouts.normal);  in rxrpc_do_sendmsg()
716 		if (p.call.timeouts.normal > 0 && j == 0)  in rxrpc_do_sendmsg()
718 		WRITE_ONCE(call->next_rx_timo, j);  in rxrpc_do_sendmsg()
721 		j = msecs_to_jiffies(p.call.timeouts.idle);  in rxrpc_do_sendmsg()
722 		if (p.call.timeouts.idle > 0 && j == 0)  in rxrpc_do_sendmsg()
724 		WRITE_ONCE(call->next_req_timo, j);  in rxrpc_do_sendmsg()
727 		if (p.call.timeouts.hard > 0) {  in rxrpc_do_sendmsg()
728 			j = msecs_to_jiffies(p.call.timeouts.hard);  in rxrpc_do_sendmsg()
731 			WRITE_ONCE(call->expect_term_by, j);  in rxrpc_do_sendmsg()
732 			rxrpc_reduce_call_timer(call, j, now,  in rxrpc_do_sendmsg()
738 	state = READ_ONCE(call->state);  in rxrpc_do_sendmsg()
739 	_debug("CALL %d USR %lx ST %d on CONN %p",  in rxrpc_do_sendmsg()
740 	       call->debug_id, call->user_call_ID, state, call->conn);  in rxrpc_do_sendmsg()
743 		/* it's too late for this call */  in rxrpc_do_sendmsg()
747 		if (rxrpc_abort_call("CMD", call, 0, p.abort_code, -ECONNABORTED))  in rxrpc_do_sendmsg()
748 			ret = rxrpc_send_abort_packet(call);  in rxrpc_do_sendmsg()
751 	} else if (rxrpc_is_client_call(call) &&  in rxrpc_do_sendmsg()
753 		/* request phase complete for this client call */  in rxrpc_do_sendmsg()
755 	} else if (rxrpc_is_service_call(call) &&  in rxrpc_do_sendmsg()
758 		/* Reply phase not begun or not complete for service call. */  in rxrpc_do_sendmsg()
761 		ret = rxrpc_send_data(rx, call, msg, len, NULL);  in rxrpc_do_sendmsg()
765 	mutex_unlock(&call->user_mutex);  in rxrpc_do_sendmsg()
767 	rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_do_sendmsg()
777  * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
778  * @sock: The socket the call is on
779  * @call: The call to send data through
784  * Allow a kernel service to send data on a call.  The call must be in an state
789 int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_send_data()  argument
795 	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);  in rxrpc_kernel_send_data()
800 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_send_data()
802 	_debug("CALL %d USR %lx ST %d on CONN %p",  in rxrpc_kernel_send_data()
803 	       call->debug_id, call->user_call_ID, call->state, call->conn);  in rxrpc_kernel_send_data()
805 	switch (READ_ONCE(call->state)) {  in rxrpc_kernel_send_data()
809 		ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,  in rxrpc_kernel_send_data()
813 		read_lock_bh(&call->state_lock);  in rxrpc_kernel_send_data()
814 		ret = call->error;  in rxrpc_kernel_send_data()
815 		read_unlock_bh(&call->state_lock);  in rxrpc_kernel_send_data()
818 		/* Request phase complete for this client call */  in rxrpc_kernel_send_data()
819 		trace_rxrpc_rx_eproto(call, 0, tracepoint_string("late_send"));  in rxrpc_kernel_send_data()
824 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_send_data()
831  * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
832  * @sock: The socket the call is on
833  * @call: The call to be aborted
838  * Allow a kernel service to abort a call, if it's still in an abortable state
839  * and return true if the call was aborted, false if it was already complete.
841 bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_abort_call()  argument
846 	_enter("{%d},%d,%d,%s", call->debug_id, abort_code, error, why);  in rxrpc_kernel_abort_call()
848 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_abort_call()
850 	aborted = rxrpc_abort_call(why, call, 0, abort_code, error);  in rxrpc_kernel_abort_call()
852 		rxrpc_send_abort_packet(call);  in rxrpc_kernel_abort_call()
854 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_abort_call()
860  * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
861  * @sock: The socket the call is on
862  * @call: The call to be informed
863  * @tx_total_len: The amount of data to be transmitted for this call
865  * Allow a kernel service to set the total transmit length on a call.  This
869  * request length can be set when beginning the call.
871 void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_set_tx_length()  argument
874 	WARN_ON(call->tx_total_len != -1);  in rxrpc_kernel_set_tx_length()
875 	call->tx_total_len = tx_total_len;  in rxrpc_kernel_set_tx_length()