kernel/bpf/stream.c

25 	 * Length denotes the amount of data to be written as part of stream element,
60 static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
62 	if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY)
64 	if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) {
65 		atomic_sub(len, &stream->capacity);
71 static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
75 	atomic_sub(len, &stream->capacity);
78 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
80 	int ret = bpf_stream_consume_capacity(stream, len);
82 	return ret ?: __bpf_stream_push_str(&stream->log, str, len);
89 	return &aux->stream[stream_id - 1];
105 static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream)
107 	return stream->backlog_head;
110 static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream)
114 	node = stream->backlog_head;
115 	if (stream->backlog_head == stream->backlog_tail)
116 		stream->backlog_head = stream->backlog_tail = NULL;
118 		stream->backlog_head = node->next;
122 static void bpf_stream_backlog_fill(struct bpf_stream *stream)
126 	if (llist_empty(&stream->log))
128 	tail = llist_del_all(&stream->log);
133 	if (!stream->backlog_head) {
134 		stream->backlog_head = head;
135 		stream->backlog_tail = tail;
137 		stream->backlog_tail->next = head;
138 		stream->backlog_tail = tail;
155 static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
161 	mutex_lock(&stream->lock);
167 		node = bpf_stream_backlog_peek(stream);
169 			bpf_stream_backlog_fill(stream);
170 			node = bpf_stream_backlog_peek(stream);
190 		bpf_stream_backlog_pop(stream);
191 		bpf_stream_release_capacity(stream, elem);
195 	mutex_unlock(&stream->lock);
201 	struct bpf_stream *stream;
203 	stream = bpf_stream_get(stream_id, prog->aux);
204 	if (!stream)
206 	return bpf_stream_read(stream, buf, len);
223 	struct bpf_stream *stream;
227 	stream = bpf_stream_get(stream_id, aux);
228 	if (!stream)
242 	ret = bpf_stream_push_str(stream, data.buf, ret);
254 	/* Make sure the stream ID is valid. */
275 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
276 		atomic_set(&prog->aux->stream[i].capacity, 0);
277 		init_llist_head(&prog->aux->stream[i].log);
278 		mutex_init(&prog->aux->stream[i].lock);
279 		prog->aux->stream[i].backlog_head = NULL;
280 		prog->aux->stream[i].backlog_tail = NULL;
289 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
290 		list = llist_del_all(&prog->aux->stream[i].log);
292 		bpf_stream_free_list(prog->aux->stream[i].backlog_head);
333 	struct bpf_stream *stream;
336 	stream = bpf_stream_get(stream_id, prog->aux);
337 	if (!stream)
340 	ret = bpf_stream_consume_capacity(stream, ss->len);
353 	llist_add_batch(head, tail, &stream->log);