Lines Matching full:zc
971 /* skb changing from pure zc to mixed, must charge zc */
1069 int zc = 0; in tcp_sendmsg_locked() local
1078 zc = MSG_ZEROCOPY; in tcp_sendmsg_locked()
1087 zc = MSG_ZEROCOPY; in tcp_sendmsg_locked()
1093 zc = MSG_SPLICE_PAGES; in tcp_sendmsg_locked()
1201 if (zc == 0) { in tcp_sendmsg_locked()
1246 } else if (zc == MSG_ZEROCOPY) { in tcp_sendmsg_locked()
1267 } else if (zc == MSG_SPLICE_PAGES) { in tcp_sendmsg_locked()
1860 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_set_hint_for_skb() argument
1868 zc->recv_skip_hint = skb->len - offset; in tcp_zerocopy_set_hint_for_skb()
1884 zc->recv_skip_hint -= partial_frag_remainder; in tcp_zerocopy_set_hint_for_skb()
1892 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); in tcp_zerocopy_set_hint_for_skb()
1893 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; in tcp_zerocopy_set_hint_for_skb()
1900 struct tcp_zerocopy_receive *zc, int inq, in receive_fallback_to_copy() argument
1903 unsigned long copy_address = (unsigned long)zc->copybuf_address; in receive_fallback_to_copy()
1907 zc->length = 0; in receive_fallback_to_copy()
1908 zc->recv_skip_hint = 0; in receive_fallback_to_copy()
1910 if (copy_address != zc->copybuf_address) in receive_fallback_to_copy()
1919 tss, &zc->msg_flags); in receive_fallback_to_copy()
1923 zc->copybuf_len = err; in receive_fallback_to_copy()
1924 if (likely(zc->copybuf_len)) { in receive_fallback_to_copy()
1930 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); in receive_fallback_to_copy()
1935 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, in tcp_copy_straggler_data() argument
1939 unsigned long copy_address = (unsigned long)zc->copybuf_address; in tcp_copy_straggler_data()
1943 if (copy_address != zc->copybuf_address) in tcp_copy_straggler_data()
1953 zc->recv_skip_hint -= copylen; in tcp_copy_straggler_data()
1959 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, in tcp_zc_handle_leftover() argument
1966 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); in tcp_zc_handle_leftover()
1977 zc->msg_flags |= TCP_CMSG_TS; in tcp_zc_handle_leftover()
1981 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, in tcp_zc_handle_leftover()
1983 return zc->copybuf_len < 0 ? 0 : copylen; in tcp_zc_handle_leftover()
1992 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_vm_insert_batch_error() argument
1998 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { in tcp_zerocopy_vm_insert_batch_error()
2029 zc->recv_skip_hint += bytes_not_mapped; in tcp_zerocopy_vm_insert_batch_error()
2040 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_vm_insert_batch() argument
2062 pages_remaining, address, length, seq, zc, total_bytes_to_map, in tcp_zerocopy_vm_insert_batch()
2068 struct tcp_zerocopy_receive *zc, in tcp_zc_finalize_rx_tstamp() argument
2074 msg_control_addr = (unsigned long)zc->msg_control; in tcp_zc_finalize_rx_tstamp()
2077 (__kernel_size_t)zc->msg_controllen; in tcp_zc_finalize_rx_tstamp()
2081 zc->msg_flags = 0; in tcp_zc_finalize_rx_tstamp()
2082 if (zc->msg_control == msg_control_addr && in tcp_zc_finalize_rx_tstamp()
2083 zc->msg_controllen == cmsg_dummy.msg_controllen) { in tcp_zc_finalize_rx_tstamp()
2085 zc->msg_control = (__u64) in tcp_zc_finalize_rx_tstamp()
2087 zc->msg_controllen = in tcp_zc_finalize_rx_tstamp()
2089 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; in tcp_zc_finalize_rx_tstamp()
2120 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_receive() argument
2124 unsigned long address = (unsigned long)zc->address; in tcp_zerocopy_receive()
2126 s32 copybuf_len = zc->copybuf_len; in tcp_zerocopy_receive()
2138 zc->copybuf_len = 0; in tcp_zerocopy_receive()
2139 zc->msg_flags = 0; in tcp_zerocopy_receive()
2141 if (address & (PAGE_SIZE - 1) || address != zc->address) in tcp_zerocopy_receive()
2150 return receive_fallback_to_copy(sk, zc, inq, tss); in tcp_zerocopy_receive()
2153 zc->length = 0; in tcp_zerocopy_receive()
2154 zc->recv_skip_hint = inq; in tcp_zerocopy_receive()
2164 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); in tcp_zerocopy_receive()
2168 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) in tcp_zerocopy_receive()
2171 zc->length = total_bytes_to_map; in tcp_zerocopy_receive()
2172 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
2174 zc->length = avail_len; in tcp_zerocopy_receive()
2175 zc->recv_skip_hint = avail_len; in tcp_zerocopy_receive()
2178 while (length + PAGE_SIZE <= zc->length) { in tcp_zerocopy_receive()
2182 if (zc->recv_skip_hint < PAGE_SIZE) { in tcp_zerocopy_receive()
2186 if (zc->recv_skip_hint > 0) in tcp_zerocopy_receive()
2199 zc->msg_flags |= TCP_CMSG_TS; in tcp_zerocopy_receive()
2201 zc->recv_skip_hint = skb->len - offset; in tcp_zerocopy_receive()
2208 zc->recv_skip_hint); in tcp_zerocopy_receive()
2210 zc->recv_skip_hint = mappable_offset; in tcp_zerocopy_receive()
2220 zc->recv_skip_hint -= PAGE_SIZE; in tcp_zerocopy_receive()
2223 zc->recv_skip_hint < PAGE_SIZE) { in tcp_zerocopy_receive()
2230 &seq, zc, in tcp_zerocopy_receive()
2240 zc, total_bytes_to_map); in tcp_zerocopy_receive()
2249 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); in tcp_zerocopy_receive()
2259 if (length == zc->length) in tcp_zerocopy_receive()
2260 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
2262 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) in tcp_zerocopy_receive()
2265 zc->length = length; in tcp_zerocopy_receive()
4606 struct tcp_zerocopy_receive zc = {}; in do_tcp_getsockopt() local
4614 if (unlikely(len > sizeof(zc))) { in do_tcp_getsockopt()
4615 err = check_zeroed_sockptr(optval, sizeof(zc), in do_tcp_getsockopt()
4616 len - sizeof(zc)); in do_tcp_getsockopt()
4619 len = sizeof(zc); in do_tcp_getsockopt()
4623 if (copy_from_sockptr(&zc, optval, len)) in do_tcp_getsockopt()
4625 if (zc.reserved) in do_tcp_getsockopt()
4627 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) in do_tcp_getsockopt()
4630 err = tcp_zerocopy_receive(sk, &zc, &tss); in do_tcp_getsockopt()
4632 &zc, &len, err); in do_tcp_getsockopt()
4653 if (zc.msg_flags & TCP_CMSG_TS) in do_tcp_getsockopt()
4654 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); in do_tcp_getsockopt()
4656 zc.msg_flags = 0; in do_tcp_getsockopt()
4659 zc.err = sock_error(sk); in do_tcp_getsockopt()
4661 zc.inq = tcp_inq_hint(sk); in do_tcp_getsockopt()
4663 if (!err && copy_to_sockptr(optval, &zc, len)) in do_tcp_getsockopt()