160 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)  in io_netmsg_iovec_free()  argument
162 	if (kmsg->vec.iovec)  in io_netmsg_iovec_free()
163 		io_vec_free(&kmsg->vec);  in io_netmsg_iovec_free()
201 				       struct io_async_msghdr *kmsg)  in io_mshot_prep_retry()  argument
352 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_setup()  local
362 	kmsg->msg.msg_name = NULL;  in io_send_setup()
363 	kmsg->msg.msg_namelen = 0;  in io_send_setup()
364 	kmsg->msg.msg_control = NULL;  in io_send_setup()
365 	kmsg->msg.msg_controllen = 0;  in io_send_setup()
366 	kmsg->msg.msg_ubuf = NULL;  in io_send_setup()
371 		ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);  in io_send_setup()
374 		kmsg->msg.msg_name = &kmsg->addr;  in io_send_setup()
375 		kmsg->msg.msg_namelen = addr_len;  in io_send_setup()
387 		return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);  in io_send_setup()
389 	return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);  in io_send_setup()
395 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg_setup()  local
400 	ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL);  in io_sendmsg_setup()
404 	sr->msg_control = kmsg->msg.msg_control_user;  in io_sendmsg_setup()
407 		kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;  in io_sendmsg_setup()
408 		return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,  in io_sendmsg_setup()
413 	return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE);  in io_sendmsg_setup()
467 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)  in io_bundle_nbufs()  argument
475 	if (iter_is_ubuf(&kmsg->msg.msg_iter))  in io_bundle_nbufs()
478 	iov = kmsg->vec.iovec;  in io_bundle_nbufs()
480 		iov = &kmsg->fast_iov;  in io_bundle_nbufs()
483 	if (!iov_iter_count(&kmsg->msg.msg_iter))  in io_bundle_nbufs()
484 		return iter_iov(&kmsg->msg.msg_iter) - iov;  in io_bundle_nbufs()
499 			      struct io_async_msghdr *kmsg, int len)  in io_net_kbuf_recyle()  argument
503 		io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));  in io_net_kbuf_recyle()
508 				  struct io_async_msghdr *kmsg,  in io_send_finish()  argument
520 	cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));  in io_send_finish()
534 		io_mshot_prep_retry(req, kmsg);  in io_send_finish()
548 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg()  local
566 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_sendmsg()
568 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg()
570 	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);  in io_sendmsg()
576 			kmsg->msg.msg_controllen = 0;  in io_sendmsg()
577 			kmsg->msg.msg_control = NULL;  in io_sendmsg()
595 				 struct io_br_sel *sel, struct io_async_msghdr *kmsg)  in io_send_select_buffer()  argument
599 		.iovs = &kmsg->fast_iov,  in io_send_select_buffer()
606 	if (kmsg->vec.iovec) {  in io_send_select_buffer()
607 		arg.nr_iovs = kmsg->vec.nr;  in io_send_select_buffer()
608 		arg.iovs = kmsg->vec.iovec;  in io_send_select_buffer()
621 	if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {  in io_send_select_buffer()
622 		kmsg->vec.nr = ret;  in io_send_select_buffer()
623 		kmsg->vec.iovec = arg.iovs;  in io_send_select_buffer()
631 					&kmsg->msg.msg_iter);  in io_send_select_buffer()
635 		iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,  in io_send_select_buffer()
645 	struct io_async_msghdr *kmsg = req->async_data;  in io_send()  local
667 		ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);  in io_send()
678 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_send()
681 	kmsg->msg.msg_flags = flags;  in io_send()
682 	ret = sock_sendmsg(sock, &kmsg->msg);  in io_send()
691 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);  in io_send()
703 	if (!io_send_finish(req, kmsg, &sel))  in io_send()
757 	struct io_async_msghdr *kmsg;  in io_recvmsg_prep_setup()  local
759 	kmsg = io_msg_alloc_async(req);  in io_recvmsg_prep_setup()
760 	if (unlikely(!kmsg))  in io_recvmsg_prep_setup()
764 		kmsg->msg.msg_name = NULL;  in io_recvmsg_prep_setup()
765 		kmsg->msg.msg_namelen = 0;  in io_recvmsg_prep_setup()
766 		kmsg->msg.msg_inq = 0;  in io_recvmsg_prep_setup()
767 		kmsg->msg.msg_control = NULL;  in io_recvmsg_prep_setup()
768 		kmsg->msg.msg_get_inq = 1;  in io_recvmsg_prep_setup()
769 		kmsg->msg.msg_controllen = 0;  in io_recvmsg_prep_setup()
770 		kmsg->msg.msg_iocb = NULL;  in io_recvmsg_prep_setup()
771 		kmsg->msg.msg_ubuf = NULL;  in io_recvmsg_prep_setup()
776 				   &kmsg->msg.msg_iter);  in io_recvmsg_prep_setup()
779 	return io_recvmsg_copy_hdr(req, kmsg);  in io_recvmsg_prep_setup()
849 				  struct io_async_msghdr *kmsg,  in io_recv_finish()  argument
856 	if (kmsg->msg.msg_inq > 0)  in io_recv_finish()
875 		cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret));  in io_recv_finish()
888 		    kmsg->msg.msg_inq > 1 && this_ret > 0 &&  in io_recv_finish()
889 		    !iov_iter_count(&kmsg->msg.msg_iter)) {  in io_recv_finish()
891 			sr->len = kmsg->msg.msg_inq;  in io_recv_finish()
907 		io_mshot_prep_retry(req, kmsg);  in io_recv_finish()
909 		if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {  in io_recv_finish()
931 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,  in io_recvmsg_prep_multishot()  argument
938 	hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +  in io_recvmsg_prep_multishot()
939 		kmsg->controllen;  in io_recvmsg_prep_multishot()
943 	if (kmsg->controllen) {  in io_recvmsg_prep_multishot()
944 		unsigned long control = ubuf + hdr - kmsg->controllen;  in io_recvmsg_prep_multishot()
946 		kmsg->msg.msg_control_user = (void __user *) control;  in io_recvmsg_prep_multishot()
947 		kmsg->msg.msg_controllen = kmsg->controllen;  in io_recvmsg_prep_multishot()
952 	kmsg->payloadlen = *len = *len - hdr;  in io_recvmsg_prep_multishot()
962 				struct io_async_msghdr *kmsg,  in io_recvmsg_multishot()  argument
969 	if (kmsg->namelen)  in io_recvmsg_multishot()
970 		kmsg->msg.msg_name = &hdr.addr;  in io_recvmsg_multishot()
971 	kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);  in io_recvmsg_multishot()
972 	kmsg->msg.msg_namelen = 0;  in io_recvmsg_multishot()
977 	err = sock_recvmsg(sock, &kmsg->msg, flags);  in io_recvmsg_multishot()
983 		.controllen = kmsg->controllen - kmsg->msg.msg_controllen,  in io_recvmsg_multishot()
984 		.flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT  in io_recvmsg_multishot()
988 	if (err > kmsg->payloadlen)  in io_recvmsg_multishot()
989 		err = kmsg->payloadlen;  in io_recvmsg_multishot()
992 	if (kmsg->msg.msg_namelen > kmsg->namelen)  in io_recvmsg_multishot()
993 		copy_len += kmsg->namelen;  in io_recvmsg_multishot()
995 		copy_len += kmsg->msg.msg_namelen;  in io_recvmsg_multishot()
1001 	hdr.msg.namelen = kmsg->msg.msg_namelen;  in io_recvmsg_multishot()
1011 	return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +  in io_recvmsg_multishot()
1012 			kmsg->controllen + err;  in io_recvmsg_multishot()
1018 	struct io_async_msghdr *kmsg = req->async_data;  in io_recvmsg()  local
1048 			ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);  in io_recvmsg()
1055 		iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);  in io_recvmsg()
1058 	kmsg->msg.msg_get_inq = 1;  in io_recvmsg()
1059 	kmsg->msg.msg_inq = -1;  in io_recvmsg()
1061 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,  in io_recvmsg()
1065 		if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)  in io_recvmsg()
1066 			min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_recvmsg()
1068 		ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,  in io_recvmsg()
1069 					 kmsg->uaddr, flags);  in io_recvmsg()
1079 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);  in io_recvmsg()
1084 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {  in io_recvmsg()
1096 	if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))  in io_recvmsg()
1102 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,  in io_recv_buf_select()  argument
1116 			.iovs = &kmsg->fast_iov,  in io_recv_buf_select()
1122 		if (kmsg->vec.iovec) {  in io_recv_buf_select()
1123 			arg.nr_iovs = kmsg->vec.nr;  in io_recv_buf_select()
1124 			arg.iovs = kmsg->vec.iovec;  in io_recv_buf_select()
1130 		else if (kmsg->msg.msg_inq > 1)  in io_recv_buf_select()
1131 			arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq);  in io_recv_buf_select()
1140 		if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {  in io_recv_buf_select()
1141 			kmsg->vec.nr = ret;  in io_recv_buf_select()
1142 			kmsg->vec.iovec = arg.iovs;  in io_recv_buf_select()
1154 		iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,  in io_recv_buf_select()
1166 				  &kmsg->msg.msg_iter);  in io_recv_buf_select()
1177 	struct io_async_msghdr *kmsg = req->async_data;  in io_recv()  local
1201 		ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);  in io_recv()
1203 			kmsg->msg.msg_inq = -1;  in io_recv()
1209 	kmsg->msg.msg_flags = 0;  in io_recv()
1210 	kmsg->msg.msg_inq = -1;  in io_recv()
1213 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_recv()
1215 	ret = sock_recvmsg(sock, &kmsg->msg, flags);  in io_recv()
1225 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);  in io_recv()
1230 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {  in io_recv()
1244 	if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))  in io_recv()
1451 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_zc_import()  local
1456 	return io_import_reg_buf(sr->notif, &kmsg->msg.msg_iter,  in io_send_zc_import()
1464 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_zc()  local
1490 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_send_zc()
1493 	kmsg->msg.msg_flags = msg_flags;  in io_send_zc()
1494 	kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;  in io_send_zc()
1495 	ret = sock_sendmsg(sock, &kmsg->msg);  in io_send_zc()
1501 		if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {  in io_send_zc()
1531 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg_zc()  local
1537 		unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;  in io_sendmsg_zc()
1541 		ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,  in io_sendmsg_zc()
1542 					sr->notif, &kmsg->vec, uvec_segs,  in io_sendmsg_zc()
1563 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_sendmsg_zc()
1565 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg_zc()
1566 	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;  in io_sendmsg_zc()
1567 	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);  in io_sendmsg_zc()
1910 	struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;  in io_netmsg_cache_free()  local
1912 	io_vec_free(&kmsg->vec);  in io_netmsg_cache_free()
1913 	kfree(kmsg);  in io_netmsg_cache_free()