1729 	struct ubuf_info_msgzc *uarg;  in msg_zerocopy_alloc()  local
1738 	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));  in msg_zerocopy_alloc()
1739 	uarg = (void *)skb->cb;  in msg_zerocopy_alloc()
1740 	uarg->mmp.user = NULL;  in msg_zerocopy_alloc()
1742 	if (likely(!devmem) && mm_account_pinned_pages(&uarg->mmp, size)) {  in msg_zerocopy_alloc()
1747 	uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;  in msg_zerocopy_alloc()
1748 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;  in msg_zerocopy_alloc()
1749 	uarg->len = 1;  in msg_zerocopy_alloc()
1750 	uarg->bytelen = size;  in msg_zerocopy_alloc()
1751 	uarg->zerocopy = 1;  in msg_zerocopy_alloc()
1752 	uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;  in msg_zerocopy_alloc()
1753 	refcount_set(&uarg->ubuf.refcnt, 1);  in msg_zerocopy_alloc()
1756 	return &uarg->ubuf;  in msg_zerocopy_alloc()
1759 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)  in skb_from_uarg()  argument
1761 	return container_of((void *)uarg, struct sk_buff, cb);  in skb_from_uarg()
1765 				       struct ubuf_info *uarg, bool devmem)  in msg_zerocopy_realloc()  argument
1767 	if (uarg) {  in msg_zerocopy_realloc()
1773 		if (uarg->ops != &msg_zerocopy_ubuf_ops)  in msg_zerocopy_realloc()
1784 		uarg_zc = uarg_to_msgzc(uarg);  in msg_zerocopy_realloc()
1804 				net_zcopy_get(uarg);  in msg_zerocopy_realloc()
1806 			return uarg;  in msg_zerocopy_realloc()
1835 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)  in __msg_zerocopy_callback()  argument
1837 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);  in __msg_zerocopy_callback()
1846 	mm_unaccount_pinned_pages(&uarg->mmp);  in __msg_zerocopy_callback()
1851 	if (!uarg->len || sock_flag(sk, SOCK_DEAD))  in __msg_zerocopy_callback()
1854 	len = uarg->len;  in __msg_zerocopy_callback()
1855 	lo = uarg->id;  in __msg_zerocopy_callback()
1856 	hi = uarg->id + len - 1;  in __msg_zerocopy_callback()
1857 	is_zerocopy = uarg->zerocopy;  in __msg_zerocopy_callback()
1885 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,  in msg_zerocopy_complete()  argument
1888 	struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);  in msg_zerocopy_complete()
1892 	if (refcount_dec_and_test(&uarg->refcnt))  in msg_zerocopy_complete()
1896 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)  in msg_zerocopy_put_abort()  argument
1898 	struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;  in msg_zerocopy_put_abort()
1901 	uarg_to_msgzc(uarg)->len--;  in msg_zerocopy_put_abort()
1904 		msg_zerocopy_complete(NULL, uarg, true);  in msg_zerocopy_put_abort()
1915 			     struct ubuf_info *uarg,  in skb_zerocopy_iter_stream()  argument
1920 	if (uarg->ops->link_skb) {  in skb_zerocopy_iter_stream()
1921 		err = uarg->ops->link_skb(skb, uarg);  in skb_zerocopy_iter_stream()
1931 		if (orig_uarg && uarg != orig_uarg)  in skb_zerocopy_iter_stream()
1948 	skb_zcopy_set(skb, uarg, NULL);  in skb_zerocopy_iter_stream()