Lines Matching full:pipe
3 * linux/fs/pipe.c
36 * New pipe buffers will be restricted to this size while the user is exceeding
37 * their pipe buffer quota. The general pipe use case needs at least two
39 * than two, then a write to a non-empty pipe may block even if the pipe is not
42 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
44 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
51 * The max size that a non-root user is allowed to grow the pipe. Can
52 * be set by root in /proc/sys/fs/pipe-max-size
89 void pipe_lock(struct pipe_inode_info *pipe) in pipe_lock() argument
91 if (pipe->files) in pipe_lock()
92 mutex_lock(&pipe->mutex); in pipe_lock()
96 void pipe_unlock(struct pipe_inode_info *pipe) in pipe_unlock() argument
98 if (pipe->files) in pipe_unlock()
99 mutex_unlock(&pipe->mutex); in pipe_unlock()
115 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe) in anon_pipe_get_page() argument
117 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in anon_pipe_get_page()
118 if (pipe->tmp_page[i]) { in anon_pipe_get_page()
119 struct page *page = pipe->tmp_page[i]; in anon_pipe_get_page()
120 pipe->tmp_page[i] = NULL; in anon_pipe_get_page()
128 static void anon_pipe_put_page(struct pipe_inode_info *pipe, in anon_pipe_put_page() argument
132 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in anon_pipe_put_page()
133 if (!pipe->tmp_page[i]) { in anon_pipe_put_page()
134 pipe->tmp_page[i] = page; in anon_pipe_put_page()
143 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, in anon_pipe_buf_release() argument
148 anon_pipe_put_page(pipe, page); in anon_pipe_buf_release()
151 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, in anon_pipe_buf_try_steal() argument
165 * @pipe: the pipe that the buffer belongs to
175 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, in generic_pipe_buf_try_steal() argument
195 * @pipe: the pipe that the buffer belongs to
201 * pipe into another.
203 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in generic_pipe_buf_get() argument
211 * @pipe: the pipe that the buffer belongs to
217 void generic_pipe_buf_release(struct pipe_inode_info *pipe, in generic_pipe_buf_release() argument
230 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
231 static inline bool pipe_readable(const struct pipe_inode_info *pipe) in pipe_readable() argument
233 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; in pipe_readable()
234 unsigned int writers = READ_ONCE(pipe->writers); in pipe_readable()
239 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe, in pipe_update_tail() argument
243 pipe_buf_release(pipe, buf); in pipe_update_tail()
246 * If the pipe has a watch_queue, we need additional protection in pipe_update_tail()
250 if (pipe_has_watch_queue(pipe)) { in pipe_update_tail()
251 spin_lock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
254 pipe->note_loss = true; in pipe_update_tail()
256 pipe->tail = ++tail; in pipe_update_tail()
257 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
265 pipe->tail = ++tail; in pipe_update_tail()
274 struct pipe_inode_info *pipe = filp->private_data; in anon_pipe_read() local
283 mutex_lock(&pipe->mutex); in anon_pipe_read()
286 * We only wake up writers if the pipe was full when we started reading in anon_pipe_read()
295 unsigned int head = smp_load_acquire(&pipe->head); in anon_pipe_read()
296 unsigned int tail = pipe->tail; in anon_pipe_read()
299 if (pipe->note_loss) { in anon_pipe_read()
318 pipe->note_loss = false; in anon_pipe_read()
323 struct pipe_buffer *buf = pipe_buf(pipe, tail); in anon_pipe_read()
337 error = pipe_buf_confirm(pipe, buf); in anon_pipe_read()
361 wake_writer |= pipe_full(head, tail, pipe->max_usage); in anon_pipe_read()
362 tail = pipe_update_tail(pipe, buf, tail); in anon_pipe_read()
371 if (!pipe->writers) in anon_pipe_read()
380 mutex_unlock(&pipe->mutex); in anon_pipe_read()
389 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) in anon_pipe_read()
393 mutex_lock(&pipe->mutex); in anon_pipe_read()
395 if (pipe_is_empty(pipe)) in anon_pipe_read()
397 mutex_unlock(&pipe->mutex); in anon_pipe_read()
400 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in anon_pipe_read()
402 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_read()
403 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in anon_pipe_read()
421 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
422 static inline bool pipe_writable(const struct pipe_inode_info *pipe) in pipe_writable() argument
424 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; in pipe_writable()
425 unsigned int max_usage = READ_ONCE(pipe->max_usage); in pipe_writable()
428 !READ_ONCE(pipe->readers); in pipe_writable()
435 struct pipe_inode_info *pipe = filp->private_data; in anon_pipe_write() local
445 * the pipe. in anon_pipe_write()
447 * pipe locked. in anon_pipe_write()
448 * If we had to support locking a normal pipe and a notification pipe at in anon_pipe_write()
452 if (pipe_has_watch_queue(pipe)) in anon_pipe_write()
459 mutex_lock(&pipe->mutex); in anon_pipe_write()
461 if (!pipe->readers) { in anon_pipe_write()
475 head = pipe->head; in anon_pipe_write()
476 was_empty = pipe_empty(head, pipe->tail); in anon_pipe_write()
479 struct pipe_buffer *buf = pipe_buf(pipe, head - 1); in anon_pipe_write()
484 ret = pipe_buf_confirm(pipe, buf); in anon_pipe_write()
501 if (!pipe->readers) { in anon_pipe_write()
508 head = pipe->head; in anon_pipe_write()
509 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { in anon_pipe_write()
514 page = anon_pipe_get_page(pipe); in anon_pipe_write()
523 anon_pipe_put_page(pipe, page); in anon_pipe_write()
529 pipe->head = head + 1; in anon_pipe_write()
531 buf = pipe_buf(pipe, head); in anon_pipe_write()
563 * We're going to release the pipe lock and wait for more in anon_pipe_write()
565 * after waiting we need to re-check whether the pipe in anon_pipe_write()
568 mutex_unlock(&pipe->mutex); in anon_pipe_write()
570 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_write()
571 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in anon_pipe_write()
572 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); in anon_pipe_write()
573 mutex_lock(&pipe->mutex); in anon_pipe_write()
574 was_empty = pipe_is_empty(pipe); in anon_pipe_write()
578 if (pipe_is_full(pipe)) in anon_pipe_write()
580 mutex_unlock(&pipe->mutex); in anon_pipe_write()
591 * Epoll nonsensically wants a wakeup whether the pipe in anon_pipe_write()
594 if (was_empty || pipe->poll_usage) in anon_pipe_write()
595 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in anon_pipe_write()
596 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in anon_pipe_write()
598 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in anon_pipe_write()
620 struct pipe_inode_info *pipe = filp->private_data; in pipe_ioctl() local
625 mutex_lock(&pipe->mutex); in pipe_ioctl()
627 head = pipe->head; in pipe_ioctl()
628 tail = pipe->tail; in pipe_ioctl()
631 count += pipe_buf(pipe, tail)->len; in pipe_ioctl()
634 mutex_unlock(&pipe->mutex); in pipe_ioctl()
641 mutex_lock(&pipe->mutex); in pipe_ioctl()
642 ret = watch_queue_set_size(pipe, arg); in pipe_ioctl()
643 mutex_unlock(&pipe->mutex); in pipe_ioctl()
649 pipe, (struct watch_notification_filter __user *)arg); in pipe_ioctl()
662 struct pipe_inode_info *pipe = filp->private_data; in pipe_poll() local
666 WRITE_ONCE(pipe->poll_usage, true); in pipe_poll()
669 * Reading pipe state only -- no need for acquiring the semaphore. in pipe_poll()
675 poll_wait(filp, &pipe->rd_wait, wait); in pipe_poll()
677 poll_wait(filp, &pipe->wr_wait, wait); in pipe_poll()
684 idx.head_tail = READ_ONCE(pipe->head_tail); in pipe_poll()
690 if (!pipe->writers && filp->f_pipe != pipe->w_counter) in pipe_poll()
695 if (!pipe_full(idx.head, idx.tail, pipe->max_usage)) in pipe_poll()
701 if (!pipe->readers) in pipe_poll()
708 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) in put_pipe_info() argument
713 if (!--pipe->files) { in put_pipe_info()
720 free_pipe_info(pipe); in put_pipe_info()
726 struct pipe_inode_info *pipe = file->private_data; in pipe_release() local
728 mutex_lock(&pipe->mutex); in pipe_release()
730 pipe->readers--; in pipe_release()
732 pipe->writers--; in pipe_release()
735 if (!pipe->readers != !pipe->writers) { in pipe_release()
736 wake_up_interruptible_all(&pipe->rd_wait); in pipe_release()
737 wake_up_interruptible_all(&pipe->wr_wait); in pipe_release()
738 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_release()
739 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_release()
741 mutex_unlock(&pipe->mutex); in pipe_release()
743 put_pipe_info(inode, pipe); in pipe_release()
750 struct pipe_inode_info *pipe = filp->private_data; in pipe_fasync() local
753 mutex_lock(&pipe->mutex); in pipe_fasync()
755 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); in pipe_fasync()
757 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); in pipe_fasync()
760 fasync_helper(-1, filp, 0, &pipe->fasync_readers); in pipe_fasync()
762 mutex_unlock(&pipe->mutex); in pipe_fasync()
793 struct pipe_inode_info *pipe; in alloc_pipe_info() local
799 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT); in alloc_pipe_info()
800 if (pipe == NULL) in alloc_pipe_info()
816 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), in alloc_pipe_info()
819 if (pipe->bufs) { in alloc_pipe_info()
820 init_waitqueue_head(&pipe->rd_wait); in alloc_pipe_info()
821 init_waitqueue_head(&pipe->wr_wait); in alloc_pipe_info()
822 pipe->r_counter = pipe->w_counter = 1; in alloc_pipe_info()
823 pipe->max_usage = pipe_bufs; in alloc_pipe_info()
824 pipe->ring_size = pipe_bufs; in alloc_pipe_info()
825 pipe->nr_accounted = pipe_bufs; in alloc_pipe_info()
826 pipe->user = user; in alloc_pipe_info()
827 mutex_init(&pipe->mutex); in alloc_pipe_info()
828 lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL); in alloc_pipe_info()
829 return pipe; in alloc_pipe_info()
834 kfree(pipe); in alloc_pipe_info()
840 void free_pipe_info(struct pipe_inode_info *pipe) in free_pipe_info() argument
845 if (pipe->watch_queue) in free_pipe_info()
846 watch_queue_clear(pipe->watch_queue); in free_pipe_info()
849 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); in free_pipe_info()
850 free_uid(pipe->user); in free_pipe_info()
851 for (i = 0; i < pipe->ring_size; i++) { in free_pipe_info()
852 struct pipe_buffer *buf = pipe->bufs + i; in free_pipe_info()
854 pipe_buf_release(pipe, buf); in free_pipe_info()
857 if (pipe->watch_queue) in free_pipe_info()
858 put_watch_queue(pipe->watch_queue); in free_pipe_info()
860 for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) { in free_pipe_info()
861 if (pipe->tmp_page[i]) in free_pipe_info()
862 __free_page(pipe->tmp_page[i]); in free_pipe_info()
864 kfree(pipe->bufs); in free_pipe_info()
865 kfree(pipe); in free_pipe_info()
875 return dynamic_dname(buffer, buflen, "pipe:[%lu]", in pipefs_dname()
888 struct pipe_inode_info *pipe; in get_pipe_inode() local
895 pipe = alloc_pipe_info(); in get_pipe_inode()
896 if (!pipe) in get_pipe_inode()
899 inode->i_pipe = pipe; in get_pipe_inode()
900 pipe->files = 2; in get_pipe_inode()
901 pipe->readers = pipe->writers = 1; in get_pipe_inode()
1001 /* pipe groks IOCB_NOWAIT */ in __do_pipe_flags()
1027 * a pipe. It's not the way Unix traditionally does this, though.
1056 SYSCALL_DEFINE1(pipe, int __user *, fildes) in SYSCALL_DEFINE1() argument
1062 * This is the stupid "wait for pipe to be readable or writable"
1070 void pipe_wait_readable(struct pipe_inode_info *pipe) in pipe_wait_readable() argument
1072 pipe_unlock(pipe); in pipe_wait_readable()
1073 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); in pipe_wait_readable()
1074 pipe_lock(pipe); in pipe_wait_readable()
1077 void pipe_wait_writable(struct pipe_inode_info *pipe) in pipe_wait_writable() argument
1079 pipe_unlock(pipe); in pipe_wait_writable()
1080 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); in pipe_wait_writable()
1081 pipe_lock(pipe); in pipe_wait_writable()
1086 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1091 * because of the pipe lock, we can check the condition before being on
1094 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1096 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) in wait_for_partner() argument
1102 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); in wait_for_partner()
1103 pipe_unlock(pipe); in wait_for_partner()
1105 finish_wait(&pipe->rd_wait, &rdwait); in wait_for_partner()
1106 pipe_lock(pipe); in wait_for_partner()
1113 static void wake_up_partner(struct pipe_inode_info *pipe) in wake_up_partner() argument
1115 wake_up_interruptible_all(&pipe->rd_wait); in wake_up_partner()
1121 struct pipe_inode_info *pipe; in fifo_open() local
1128 pipe = inode->i_pipe; in fifo_open()
1129 pipe->files++; in fifo_open()
1133 pipe = alloc_pipe_info(); in fifo_open()
1134 if (!pipe) in fifo_open()
1136 pipe->files = 1; in fifo_open()
1141 free_pipe_info(pipe); in fifo_open()
1142 pipe = inode->i_pipe; in fifo_open()
1144 inode->i_pipe = pipe; in fifo_open()
1148 filp->private_data = pipe; in fifo_open()
1149 /* OK, we have a pipe and it's pinned down */ in fifo_open()
1151 mutex_lock(&pipe->mutex); in fifo_open()
1163 pipe->r_counter++; in fifo_open()
1164 if (pipe->readers++ == 0) in fifo_open()
1165 wake_up_partner(pipe); in fifo_open()
1167 if (!is_pipe && !pipe->writers) { in fifo_open()
1171 filp->f_pipe = pipe->w_counter; in fifo_open()
1173 if (wait_for_partner(pipe, &pipe->w_counter)) in fifo_open()
1186 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) in fifo_open()
1189 pipe->w_counter++; in fifo_open()
1190 if (!pipe->writers++) in fifo_open()
1191 wake_up_partner(pipe); in fifo_open()
1193 if (!is_pipe && !pipe->readers) { in fifo_open()
1194 if (wait_for_partner(pipe, &pipe->r_counter)) in fifo_open()
1207 pipe->readers++; in fifo_open()
1208 pipe->writers++; in fifo_open()
1209 pipe->r_counter++; in fifo_open()
1210 pipe->w_counter++; in fifo_open()
1211 if (pipe->readers == 1 || pipe->writers == 1) in fifo_open()
1212 wake_up_partner(pipe); in fifo_open()
1221 mutex_unlock(&pipe->mutex); in fifo_open()
1225 if (!--pipe->readers) in fifo_open()
1226 wake_up_interruptible(&pipe->wr_wait); in fifo_open()
1231 if (!--pipe->writers) in fifo_open()
1232 wake_up_interruptible_all(&pipe->rd_wait); in fifo_open()
1237 mutex_unlock(&pipe->mutex); in fifo_open()
1239 put_pipe_info(inode, pipe); in fifo_open()
1266 * Currently we rely on the pipe array holding a power-of-2 number
1274 /* Minimum pipe size, as required by POSIX */ in round_pipe_size()
1282 * Resize the pipe ring to a number of slots.
1284 * Note the pipe can be reduced in capacity, but only if the current
1288 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) in pipe_resize_ring() argument
1293 /* nr_slots larger than limits of pipe->{head,tail} */ in pipe_resize_ring()
1302 spin_lock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1303 mask = pipe->ring_size - 1; in pipe_resize_ring()
1304 head = pipe->head; in pipe_resize_ring()
1305 tail = pipe->tail; in pipe_resize_ring()
1309 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1315 * The pipe array wraps around, so just start the new one at zero in pipe_resize_ring()
1322 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1325 unsigned int tsize = pipe->ring_size - t; in pipe_resize_ring()
1327 memcpy(bufs + tsize, pipe->bufs, in pipe_resize_ring()
1329 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1337 kfree(pipe->bufs); in pipe_resize_ring()
1338 pipe->bufs = bufs; in pipe_resize_ring()
1339 pipe->ring_size = nr_slots; in pipe_resize_ring()
1340 if (pipe->max_usage > nr_slots) in pipe_resize_ring()
1341 pipe->max_usage = nr_slots; in pipe_resize_ring()
1342 pipe->tail = tail; in pipe_resize_ring()
1343 pipe->head = head; in pipe_resize_ring()
1345 if (!pipe_has_watch_queue(pipe)) { in pipe_resize_ring()
1346 pipe->max_usage = nr_slots; in pipe_resize_ring()
1347 pipe->nr_accounted = nr_slots; in pipe_resize_ring()
1350 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1353 wake_up_interruptible(&pipe->wr_wait); in pipe_resize_ring()
1358 * Allocate a new array of pipe buffers and copy the info over. Returns the
1359 * pipe size if successful, or return -ERROR on error.
1361 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg) in pipe_set_size() argument
1367 if (pipe_has_watch_queue(pipe)) in pipe_set_size()
1377 * If trying to increase the pipe capacity, check that an in pipe_set_size()
1380 * Decreasing the pipe capacity is always permitted, even in pipe_set_size()
1383 if (nr_slots > pipe->max_usage && in pipe_set_size()
1387 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); in pipe_set_size()
1389 if (nr_slots > pipe->max_usage && in pipe_set_size()
1397 ret = pipe_resize_ring(pipe, nr_slots); in pipe_set_size()
1401 return pipe->max_usage * PAGE_SIZE; in pipe_set_size()
1404 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); in pipe_set_size()
1410 * not enough to verify that this is a pipe.
1414 struct pipe_inode_info *pipe = file->private_data; in get_pipe_info() local
1416 if (!pipe) in get_pipe_info()
1420 if (for_splice && pipe_has_watch_queue(pipe)) in get_pipe_info()
1422 return pipe; in get_pipe_info()
1427 struct pipe_inode_info *pipe; in pipe_fcntl() local
1430 pipe = get_pipe_info(file, false); in pipe_fcntl()
1431 if (!pipe) in pipe_fcntl()
1434 mutex_lock(&pipe->mutex); in pipe_fcntl()
1438 ret = pipe_set_size(pipe, arg); in pipe_fcntl()
1441 ret = pipe->max_usage * PAGE_SIZE; in pipe_fcntl()
1448 mutex_unlock(&pipe->mutex); in pipe_fcntl()
1461 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1510 .procname = "pipe-max-size",
1517 .procname = "pipe-user-pages-hard",
1524 .procname = "pipe-user-pages-soft",