118 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
122 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
123 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
124 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
125 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
126 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
129 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
130 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
133 			      struct sk_msg *msg, u32 bytes);
135 			     struct sk_msg *msg, u32 bytes);
136 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
140 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
142 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
175 #define sk_msg_iter_prev(msg, which)			\  argument
176 	sk_msg_iter_var_prev(msg->sg.which)
178 #define sk_msg_iter_next(msg, which)			\  argument
179 	sk_msg_iter_var_next(msg->sg.which)
181 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
183 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);  in sk_msg_init()
184 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
185 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in sk_msg_init()
205 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
207 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;  in sk_msg_full()
210 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
212 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
215 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
217 	return &msg->sg.data[which];  in sk_msg_elem()
220 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
222 	return msg->sg.data[which];  in sk_msg_elem_cpy()
225 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
227 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
230 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
232 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
235 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
237 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
239 	if (test_bit(msg->sg.start, msg->sg.copy)) {  in sk_msg_compute_data_pointers()
240 		msg->data = NULL;  in sk_msg_compute_data_pointers()
241 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
243 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
244 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
248 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
254 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
258 	__set_bit(msg->sg.end, msg->sg.copy);  in sk_msg_page_add()
259 	msg->sg.size += len;  in sk_msg_page_add()
260 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
263 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
267 			__set_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
269 			__clear_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
271 		if (i == msg->sg.end)  in sk_msg_sg_copy()
276 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
278 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
281 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
283 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
317 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
321 		list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
323 		sk_msg_free(psock->sk, msg);  in sk_psock_queue_msg()
324 		kfree(msg);  in sk_psock_queue_msg()
331 	struct sk_msg *msg;  in sk_psock_dequeue_msg()  local
334 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_dequeue_msg()
335 	if (msg)  in sk_psock_dequeue_msg()
336 		list_del(&msg->list);  in sk_psock_dequeue_msg()
338 	return msg;  in sk_psock_dequeue_msg()
343 	struct sk_msg *msg;  in sk_psock_peek_msg()  local
346 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_peek_msg()
348 	return msg;  in sk_psock_peek_msg()
352 					       struct sk_msg *msg)  in sk_psock_next_msg()  argument
357 	if (list_is_last(&msg->list, &psock->ingress_msg))  in sk_psock_next_msg()
360 		ret = list_next_entry(msg, list);  in sk_psock_next_msg()
370 static inline void kfree_sk_msg(struct sk_msg *msg)  in kfree_sk_msg()  argument
372 	if (msg->skb)  in kfree_sk_msg()
373 		consume_skb(msg->skb);  in kfree_sk_msg()
374 	kfree(msg);  in kfree_sk_msg()
411 			 struct sk_msg *msg);