20  * Post a call for attention by the socket or kernel service.  Further
23 void rxrpc_notify_socket(struct rxrpc_call *call)  in rxrpc_notify_socket()  argument
28 	_enter("%d", call->debug_id);  in rxrpc_notify_socket()
30 	if (!list_empty(&call->recvmsg_link))  in rxrpc_notify_socket()
35 	rx = rcu_dereference(call->socket);  in rxrpc_notify_socket()
38 		if (call->notify_rx) {  in rxrpc_notify_socket()
39 			spin_lock_bh(&call->notify_lock);  in rxrpc_notify_socket()
40 			call->notify_rx(sk, call, call->user_call_ID);  in rxrpc_notify_socket()
41 			spin_unlock_bh(&call->notify_lock);  in rxrpc_notify_socket()
44 			if (list_empty(&call->recvmsg_link)) {  in rxrpc_notify_socket()
45 				rxrpc_get_call(call, rxrpc_call_got);  in rxrpc_notify_socket()
46 				list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);  in rxrpc_notify_socket()
51 				_debug("call %ps", sk->sk_data_ready);  in rxrpc_notify_socket()
62  * Transition a call to the complete state.
64 bool __rxrpc_set_call_completion(struct rxrpc_call *call,  in __rxrpc_set_call_completion()  argument
69 	if (call->state < RXRPC_CALL_COMPLETE) {  in __rxrpc_set_call_completion()
70 		call->abort_code = abort_code;  in __rxrpc_set_call_completion()
71 		call->error = error;  in __rxrpc_set_call_completion()
72 		call->completion = compl,  in __rxrpc_set_call_completion()
73 		call->state = RXRPC_CALL_COMPLETE;  in __rxrpc_set_call_completion()
74 		trace_rxrpc_call_complete(call);  in __rxrpc_set_call_completion()
75 		wake_up(&call->waitq);  in __rxrpc_set_call_completion()
76 		rxrpc_notify_socket(call);  in __rxrpc_set_call_completion()
82 bool rxrpc_set_call_completion(struct rxrpc_call *call,  in rxrpc_set_call_completion()  argument
89 	if (call->state < RXRPC_CALL_COMPLETE) {  in rxrpc_set_call_completion()
90 		write_lock_bh(&call->state_lock);  in rxrpc_set_call_completion()
91 		ret = __rxrpc_set_call_completion(call, compl, abort_code, error);  in rxrpc_set_call_completion()
92 		write_unlock_bh(&call->state_lock);  in rxrpc_set_call_completion()
98  * Record that a call successfully completed.
100 bool __rxrpc_call_completed(struct rxrpc_call *call)  in __rxrpc_call_completed()  argument
102 	return __rxrpc_set_call_completion(call, RXRPC_CALL_SUCCEEDED, 0, 0);  in __rxrpc_call_completed()
105 bool rxrpc_call_completed(struct rxrpc_call *call)  in rxrpc_call_completed()  argument
109 	if (call->state < RXRPC_CALL_COMPLETE) {  in rxrpc_call_completed()
110 		write_lock_bh(&call->state_lock);  in rxrpc_call_completed()
111 		ret = __rxrpc_call_completed(call);  in rxrpc_call_completed()
112 		write_unlock_bh(&call->state_lock);  in rxrpc_call_completed()
118  * Record that a call is locally aborted.
120 bool __rxrpc_abort_call(const char *why, struct rxrpc_call *call,  in __rxrpc_abort_call()  argument
123 	trace_rxrpc_abort(call->debug_id, why, call->cid, call->call_id, seq,  in __rxrpc_abort_call()
125 	return __rxrpc_set_call_completion(call, RXRPC_CALL_LOCALLY_ABORTED,  in __rxrpc_abort_call()
129 bool rxrpc_abort_call(const char *why, struct rxrpc_call *call,  in rxrpc_abort_call()  argument
134 	write_lock_bh(&call->state_lock);  in rxrpc_abort_call()
135 	ret = __rxrpc_abort_call(why, call, seq, abort_code, error);  in rxrpc_abort_call()
136 	write_unlock_bh(&call->state_lock);  in rxrpc_abort_call()
141  * Pass a call terminating message to userspace.
143 static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)  in rxrpc_recvmsg_term()  argument
148 	switch (call->completion) {  in rxrpc_recvmsg_term()
151 		if (rxrpc_is_service_call(call))  in rxrpc_recvmsg_term()
155 		tmp = call->abort_code;  in rxrpc_recvmsg_term()
159 		tmp = call->abort_code;  in rxrpc_recvmsg_term()
163 		tmp = -call->error;  in rxrpc_recvmsg_term()
167 		tmp = -call->error;  in rxrpc_recvmsg_term()
171 		pr_err("Invalid terminal call state %u\n", call->state);  in rxrpc_recvmsg_term()
176 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_terminal, call->rx_hard_ack,  in rxrpc_recvmsg_term()
177 			    call->rx_pkt_offset, call->rx_pkt_len, ret);  in rxrpc_recvmsg_term()
184 static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial)  in rxrpc_end_rx_phase()  argument
186 	_enter("%d,%s", call->debug_id, rxrpc_call_states[call->state]);  in rxrpc_end_rx_phase()
188 	trace_rxrpc_receive(call, rxrpc_receive_end, 0, call->rx_top);  in rxrpc_end_rx_phase()
189 	ASSERTCMP(call->rx_hard_ack, ==, call->rx_top);  in rxrpc_end_rx_phase()
191 	if (call->state == RXRPC_CALL_CLIENT_RECV_REPLY) {  in rxrpc_end_rx_phase()
192 		rxrpc_propose_ACK(call, RXRPC_ACK_IDLE, serial, false, true,  in rxrpc_end_rx_phase()
194 		//rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_end_rx_phase()
197 	write_lock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
199 	switch (call->state) {  in rxrpc_end_rx_phase()
201 		__rxrpc_call_completed(call);  in rxrpc_end_rx_phase()
202 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
206 		call->tx_phase = true;  in rxrpc_end_rx_phase()
207 		call->state = RXRPC_CALL_SERVER_ACK_REQUEST;  in rxrpc_end_rx_phase()
208 		call->expect_req_by = jiffies + MAX_JIFFY_OFFSET;  in rxrpc_end_rx_phase()
209 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
210 		rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial, false, true,  in rxrpc_end_rx_phase()
214 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
222 static void rxrpc_rotate_rx_window(struct rxrpc_call *call)  in rxrpc_rotate_rx_window()  argument
232 	_enter("%d", call->debug_id);  in rxrpc_rotate_rx_window()
234 	hard_ack = call->rx_hard_ack;  in rxrpc_rotate_rx_window()
235 	top = smp_load_acquire(&call->rx_top);  in rxrpc_rotate_rx_window()
240 	skb = call->rxtx_buffer[ix];  in rxrpc_rotate_rx_window()
244 	subpacket = call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;  in rxrpc_rotate_rx_window()
251 	call->rxtx_buffer[ix] = NULL;  in rxrpc_rotate_rx_window()
252 	call->rxtx_annotations[ix] = 0;  in rxrpc_rotate_rx_window()
254 	smp_store_release(&call->rx_hard_ack, hard_ack);  in rxrpc_rotate_rx_window()
258 	trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);  in rxrpc_rotate_rx_window()
260 		rxrpc_end_rx_phase(call, serial);  in rxrpc_rotate_rx_window()
263 		if (after_eq(hard_ack, call->ackr_consumed + 2) ||  in rxrpc_rotate_rx_window()
264 		    after_eq(top, call->ackr_seen + 2) ||  in rxrpc_rotate_rx_window()
265 		    (hard_ack == top && after(hard_ack, call->ackr_consumed)))  in rxrpc_rotate_rx_window()
266 			rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial,  in rxrpc_rotate_rx_window()
269 		if (call->ackr_reason && call->ackr_reason != RXRPC_ACK_DELAY)  in rxrpc_rotate_rx_window()
270 			rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_rotate_rx_window()
280 static int rxrpc_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,  in rxrpc_verify_packet()  argument
302 	return call->security->verify_packet(call, skb, offset, len,  in rxrpc_verify_packet()
316 static int rxrpc_locate_data(struct rxrpc_call *call, struct sk_buff *skb,  in rxrpc_locate_data()  argument
338 		ret = rxrpc_verify_packet(call, skb, annotation, offset, len);  in rxrpc_locate_data()
347 	call->security->locate_data(call, skb, _offset, _len);  in rxrpc_locate_data()
352  * Deliver messages to a call.  This keeps processing packets until the buffer
356 static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_recvmsg_data()  argument
369 	if (test_and_clear_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags) &&  in rxrpc_recvmsg_data()
370 	    call->ackr_reason)  in rxrpc_recvmsg_data()
371 		rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_recvmsg_data()
373 	rx_pkt_offset = call->rx_pkt_offset;  in rxrpc_recvmsg_data()
374 	rx_pkt_len = call->rx_pkt_len;  in rxrpc_recvmsg_data()
375 	rx_pkt_last = call->rx_pkt_last;  in rxrpc_recvmsg_data()
377 	if (call->state >= RXRPC_CALL_SERVER_ACK_REQUEST) {  in rxrpc_recvmsg_data()
378 		seq = call->rx_hard_ack;  in rxrpc_recvmsg_data()
384 	hard_ack = call->rx_hard_ack;  in rxrpc_recvmsg_data()
387 	while (top = smp_load_acquire(&call->rx_top),  in rxrpc_recvmsg_data()
391 		skb = call->rxtx_buffer[ix];  in rxrpc_recvmsg_data()
393 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_hole, seq,  in rxrpc_recvmsg_data()
403 			serial += call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;  in rxrpc_recvmsg_data()
404 			trace_rxrpc_receive(call, rxrpc_receive_front,  in rxrpc_recvmsg_data()
412 			ret2 = rxrpc_locate_data(call, skb,  in rxrpc_recvmsg_data()
413 						 &call->rxtx_annotations[ix],  in rxrpc_recvmsg_data()
416 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_next, seq,  in rxrpc_recvmsg_data()
423 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_cont, seq,  in rxrpc_recvmsg_data()
447 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_full, seq,  in rxrpc_recvmsg_data()
456 			rxrpc_rotate_rx_window(call);  in rxrpc_recvmsg_data()
461 			ASSERTCMP(seq, ==, READ_ONCE(call->rx_top));  in rxrpc_recvmsg_data()
471 		call->rx_pkt_offset = rx_pkt_offset;  in rxrpc_recvmsg_data()
472 		call->rx_pkt_len = rx_pkt_len;  in rxrpc_recvmsg_data()
473 		call->rx_pkt_last = rx_pkt_last;  in rxrpc_recvmsg_data()
476 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_data_return, seq,  in rxrpc_recvmsg_data()
479 		set_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags);  in rxrpc_recvmsg_data()
491 	struct rxrpc_call *call;  in rxrpc_recvmsg()  local
521 			call = NULL;  in rxrpc_recvmsg()
545 	/* Find the next call and dequeue it if we're not just peeking.  If we  in rxrpc_recvmsg()
550 	call = list_entry(l, struct rxrpc_call, recvmsg_link);  in rxrpc_recvmsg()
552 		list_del_init(&call->recvmsg_link);  in rxrpc_recvmsg()
554 		rxrpc_get_call(call, rxrpc_call_got);  in rxrpc_recvmsg()
557 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0, 0, 0, 0);  in rxrpc_recvmsg()
559 	/* We're going to drop the socket lock, so we need to lock the call  in rxrpc_recvmsg()
562 	if (!mutex_trylock(&call->user_mutex)) {  in rxrpc_recvmsg()
567 		if (mutex_lock_interruptible(&call->user_mutex) < 0)  in rxrpc_recvmsg()
573 	if (test_bit(RXRPC_CALL_RELEASED, &call->flags))  in rxrpc_recvmsg()
576 	if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {  in rxrpc_recvmsg()
578 			unsigned int id32 = call->user_call_ID;  in rxrpc_recvmsg()
583 			unsigned long idl = call->user_call_ID;  in rxrpc_recvmsg()
592 	if (msg->msg_name && call->peer) {  in rxrpc_recvmsg()
594 		size_t len = sizeof(call->peer->srx);  in rxrpc_recvmsg()
596 		memcpy(msg->msg_name, &call->peer->srx, len);  in rxrpc_recvmsg()
597 		srx->srx_service = call->service_id;  in rxrpc_recvmsg()
601 	switch (READ_ONCE(call->state)) {  in rxrpc_recvmsg()
605 		ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,  in rxrpc_recvmsg()
610 		if (after(call->rx_top, call->rx_hard_ack) &&  in rxrpc_recvmsg()
611 		    call->rxtx_buffer[(call->rx_hard_ack + 1) & RXRPC_RXTX_BUFF_MASK])  in rxrpc_recvmsg()
612 			rxrpc_notify_socket(call);  in rxrpc_recvmsg()
622 	if (call->state == RXRPC_CALL_COMPLETE) {  in rxrpc_recvmsg()
623 		ret = rxrpc_recvmsg_term(call, msg);  in rxrpc_recvmsg()
627 			rxrpc_release_call(rx, call);  in rxrpc_recvmsg()
639 	mutex_unlock(&call->user_mutex);  in rxrpc_recvmsg()
640 	rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_recvmsg()
641 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);  in rxrpc_recvmsg()
647 		list_add(&call->recvmsg_link, &rx->recvmsg_q);  in rxrpc_recvmsg()
649 		trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0, 0, 0, 0);  in rxrpc_recvmsg()
651 		rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_recvmsg()
656 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);  in rxrpc_recvmsg()
663 	call = NULL;  in rxrpc_recvmsg()
669  * @sock: The socket that the call exists on
670  * @call: The call to send data through
677  * state of a call.  Returns 0 if got what was asked for and there's more
686 int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_recv_data()  argument
694 	       call->debug_id, rxrpc_call_states[call->state],  in rxrpc_kernel_recv_data()
697 	ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_SECURING);  in rxrpc_kernel_recv_data()
699 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_recv_data()
701 	switch (READ_ONCE(call->state)) {  in rxrpc_kernel_recv_data()
705 		ret = rxrpc_recvmsg_data(sock, call, NULL, iter,  in rxrpc_kernel_recv_data()
739 	switch (call->ackr_reason) {  in rxrpc_kernel_recv_data()
747 		rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_kernel_recv_data()
751 		*_service = call->service_id;  in rxrpc_kernel_recv_data()
752 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_recv_data()
757 	trace_rxrpc_rx_eproto(call, 0, tracepoint_string("short_data"));  in rxrpc_kernel_recv_data()
761 	trace_rxrpc_rx_eproto(call, 0, tracepoint_string("excess_data"));  in rxrpc_kernel_recv_data()
765 	*_abort = call->abort_code;  in rxrpc_kernel_recv_data()
766 	ret = call->error;  in rxrpc_kernel_recv_data()
767 	if (call->completion == RXRPC_CALL_SUCCEEDED) {  in rxrpc_kernel_recv_data()
778  * @sock: The socket that the call exists on
779  * @call: The call to query
785 bool rxrpc_kernel_get_reply_time(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_get_reply_time()  argument
792 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_get_reply_time()
794 	if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_RECV_REPLY)  in rxrpc_kernel_get_reply_time()
797 	hard_ack = call->rx_hard_ack;  in rxrpc_kernel_get_reply_time()
802 	top = smp_load_acquire(&call->rx_top);  in rxrpc_kernel_get_reply_time()
806 	skb = call->rxtx_buffer[seq & RXRPC_RXTX_BUFF_MASK];  in rxrpc_kernel_get_reply_time()
814 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_get_reply_time()