Lines Matching full:pipe
3 * linux/fs/pipe.c
36 * New pipe buffers will be restricted to this size while the user is exceeding
37 * their pipe buffer quota. The general pipe use case needs at least two
39 * than two, then a write to a non-empty pipe may block even if the pipe is not
42 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
44 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
51 * The max size that a non-root user is allowed to grow the pipe. Can
52 * be set by root in /proc/sys/fs/pipe-max-size
79 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass) in pipe_lock_nested() argument
81 if (pipe->files) in pipe_lock_nested()
82 mutex_lock_nested(&pipe->mutex, subclass); in pipe_lock_nested()
85 void pipe_lock(struct pipe_inode_info *pipe) in pipe_lock() argument
88 * pipe_lock() nests non-pipe inode locks (for writing to a file) in pipe_lock()
90 pipe_lock_nested(pipe, I_MUTEX_PARENT); in pipe_lock()
94 void pipe_unlock(struct pipe_inode_info *pipe) in pipe_unlock() argument
96 if (pipe->files) in pipe_unlock()
97 mutex_unlock(&pipe->mutex); in pipe_unlock()
101 static inline void __pipe_lock(struct pipe_inode_info *pipe) in __pipe_lock() argument
103 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT); in __pipe_lock()
106 static inline void __pipe_unlock(struct pipe_inode_info *pipe) in __pipe_unlock() argument
108 mutex_unlock(&pipe->mutex); in __pipe_unlock()
125 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, in anon_pipe_buf_release() argument
135 if (page_count(page) == 1 && !pipe->tmp_page) in anon_pipe_buf_release()
136 pipe->tmp_page = page; in anon_pipe_buf_release()
141 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, in anon_pipe_buf_try_steal() argument
155 * @pipe: the pipe that the buffer belongs to
165 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, in generic_pipe_buf_try_steal() argument
185 * @pipe: the pipe that the buffer belongs to
191 * pipe into another.
193 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in generic_pipe_buf_get() argument
201 * @pipe: the pipe that the buffer belongs to
207 void generic_pipe_buf_release(struct pipe_inode_info *pipe, in generic_pipe_buf_release() argument
220 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
221 static inline bool pipe_readable(const struct pipe_inode_info *pipe) in pipe_readable() argument
223 unsigned int head = READ_ONCE(pipe->head); in pipe_readable()
224 unsigned int tail = READ_ONCE(pipe->tail); in pipe_readable()
225 unsigned int writers = READ_ONCE(pipe->writers); in pipe_readable()
230 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe, in pipe_update_tail() argument
234 pipe_buf_release(pipe, buf); in pipe_update_tail()
237 * If the pipe has a watch_queue, we need additional protection in pipe_update_tail()
241 if (pipe_has_watch_queue(pipe)) { in pipe_update_tail()
242 spin_lock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
245 pipe->note_loss = true; in pipe_update_tail()
247 pipe->tail = ++tail; in pipe_update_tail()
248 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_update_tail()
256 pipe->tail = ++tail; in pipe_update_tail()
265 struct pipe_inode_info *pipe = filp->private_data; in pipe_read() local
274 __pipe_lock(pipe); in pipe_read()
277 * We only wake up writers if the pipe was full when we started in pipe_read()
284 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
287 unsigned int head = smp_load_acquire(&pipe->head); in pipe_read()
288 unsigned int tail = pipe->tail; in pipe_read()
289 unsigned int mask = pipe->ring_size - 1; in pipe_read()
292 if (pipe->note_loss) { in pipe_read()
311 pipe->note_loss = false; in pipe_read()
316 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in pipe_read()
330 error = pipe_buf_confirm(pipe, buf); in pipe_read()
354 tail = pipe_update_tail(pipe, buf, tail); in pipe_read()
362 if (!pipe->writers) in pipe_read()
371 __pipe_unlock(pipe); in pipe_read()
377 * pipe buffer, and might have made space in the buffers in pipe_read()
380 * You can't make zero-sized pipe buffers by doing an empty in pipe_read()
387 * _very_ unlikely case that the pipe was full, but we got in pipe_read()
391 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
392 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
400 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) in pipe_read()
403 __pipe_lock(pipe); in pipe_read()
404 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
407 if (pipe_empty(pipe->head, pipe->tail)) in pipe_read()
409 __pipe_unlock(pipe); in pipe_read()
412 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
414 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_read()
415 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
426 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
427 static inline bool pipe_writable(const struct pipe_inode_info *pipe) in pipe_writable() argument
429 unsigned int head = READ_ONCE(pipe->head); in pipe_writable()
430 unsigned int tail = READ_ONCE(pipe->tail); in pipe_writable()
431 unsigned int max_usage = READ_ONCE(pipe->max_usage); in pipe_writable()
434 !READ_ONCE(pipe->readers); in pipe_writable()
441 struct pipe_inode_info *pipe = filp->private_data; in pipe_write() local
451 * the pipe. in pipe_write()
453 * pipe locked. in pipe_write()
454 * If we had to support locking a normal pipe and a notification pipe at in pipe_write()
458 if (pipe_has_watch_queue(pipe)) in pipe_write()
465 __pipe_lock(pipe); in pipe_write()
467 if (!pipe->readers) { in pipe_write()
481 head = pipe->head; in pipe_write()
482 was_empty = pipe_empty(head, pipe->tail); in pipe_write()
485 unsigned int mask = pipe->ring_size - 1; in pipe_write()
486 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask]; in pipe_write()
491 ret = pipe_buf_confirm(pipe, buf); in pipe_write()
508 if (!pipe->readers) { in pipe_write()
515 head = pipe->head; in pipe_write()
516 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { in pipe_write()
517 unsigned int mask = pipe->ring_size - 1; in pipe_write()
519 struct page *page = pipe->tmp_page; in pipe_write()
528 pipe->tmp_page = page; in pipe_write()
536 pipe->head = head + 1; in pipe_write()
539 buf = &pipe->bufs[head & mask]; in pipe_write()
548 pipe->tmp_page = NULL; in pipe_write()
563 if (!pipe_full(head, pipe->tail, pipe->max_usage)) in pipe_write()
580 * We're going to release the pipe lock and wait for more in pipe_write()
582 * after waiting we need to re-check whether the pipe in pipe_write()
585 __pipe_unlock(pipe); in pipe_write()
587 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
588 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
589 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); in pipe_write()
590 __pipe_lock(pipe); in pipe_write()
591 was_empty = pipe_empty(pipe->head, pipe->tail); in pipe_write()
595 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in pipe_write()
597 __pipe_unlock(pipe); in pipe_write()
608 * Epoll nonsensically wants a wakeup whether the pipe in pipe_write()
611 if (was_empty || pipe->poll_usage) in pipe_write()
612 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
613 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
615 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_write()
627 struct pipe_inode_info *pipe = filp->private_data; in pipe_ioctl() local
632 __pipe_lock(pipe); in pipe_ioctl()
634 head = pipe->head; in pipe_ioctl()
635 tail = pipe->tail; in pipe_ioctl()
636 mask = pipe->ring_size - 1; in pipe_ioctl()
639 count += pipe->bufs[tail & mask].len; in pipe_ioctl()
642 __pipe_unlock(pipe); in pipe_ioctl()
649 __pipe_lock(pipe); in pipe_ioctl()
650 ret = watch_queue_set_size(pipe, arg); in pipe_ioctl()
651 __pipe_unlock(pipe); in pipe_ioctl()
657 pipe, (struct watch_notification_filter __user *)arg); in pipe_ioctl()
670 struct pipe_inode_info *pipe = filp->private_data; in pipe_poll() local
674 WRITE_ONCE(pipe->poll_usage, true); in pipe_poll()
677 * Reading pipe state only -- no need for acquiring the semaphore. in pipe_poll()
683 poll_wait(filp, &pipe->rd_wait, wait); in pipe_poll()
685 poll_wait(filp, &pipe->wr_wait, wait); in pipe_poll()
692 head = READ_ONCE(pipe->head); in pipe_poll()
693 tail = READ_ONCE(pipe->tail); in pipe_poll()
699 if (!pipe->writers && filp->f_version != pipe->w_counter) in pipe_poll()
704 if (!pipe_full(head, tail, pipe->max_usage)) in pipe_poll()
710 if (!pipe->readers) in pipe_poll()
717 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) in put_pipe_info() argument
722 if (!--pipe->files) { in put_pipe_info()
729 free_pipe_info(pipe); in put_pipe_info()
735 struct pipe_inode_info *pipe = file->private_data; in pipe_release() local
737 __pipe_lock(pipe); in pipe_release()
739 pipe->readers--; in pipe_release()
741 pipe->writers--; in pipe_release()
744 if (!pipe->readers != !pipe->writers) { in pipe_release()
745 wake_up_interruptible_all(&pipe->rd_wait); in pipe_release()
746 wake_up_interruptible_all(&pipe->wr_wait); in pipe_release()
747 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_release()
748 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_release()
750 __pipe_unlock(pipe); in pipe_release()
752 put_pipe_info(inode, pipe); in pipe_release()
759 struct pipe_inode_info *pipe = filp->private_data; in pipe_fasync() local
762 __pipe_lock(pipe); in pipe_fasync()
764 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); in pipe_fasync()
766 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); in pipe_fasync()
769 fasync_helper(-1, filp, 0, &pipe->fasync_readers); in pipe_fasync()
771 __pipe_unlock(pipe); in pipe_fasync()
802 struct pipe_inode_info *pipe; in alloc_pipe_info() local
808 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT); in alloc_pipe_info()
809 if (pipe == NULL) in alloc_pipe_info()
825 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), in alloc_pipe_info()
828 if (pipe->bufs) { in alloc_pipe_info()
829 init_waitqueue_head(&pipe->rd_wait); in alloc_pipe_info()
830 init_waitqueue_head(&pipe->wr_wait); in alloc_pipe_info()
831 pipe->r_counter = pipe->w_counter = 1; in alloc_pipe_info()
832 pipe->max_usage = pipe_bufs; in alloc_pipe_info()
833 pipe->ring_size = pipe_bufs; in alloc_pipe_info()
834 pipe->nr_accounted = pipe_bufs; in alloc_pipe_info()
835 pipe->user = user; in alloc_pipe_info()
836 mutex_init(&pipe->mutex); in alloc_pipe_info()
837 return pipe; in alloc_pipe_info()
842 kfree(pipe); in alloc_pipe_info()
848 void free_pipe_info(struct pipe_inode_info *pipe) in free_pipe_info() argument
853 if (pipe->watch_queue) in free_pipe_info()
854 watch_queue_clear(pipe->watch_queue); in free_pipe_info()
857 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); in free_pipe_info()
858 free_uid(pipe->user); in free_pipe_info()
859 for (i = 0; i < pipe->ring_size; i++) { in free_pipe_info()
860 struct pipe_buffer *buf = pipe->bufs + i; in free_pipe_info()
862 pipe_buf_release(pipe, buf); in free_pipe_info()
865 if (pipe->watch_queue) in free_pipe_info()
866 put_watch_queue(pipe->watch_queue); in free_pipe_info()
868 if (pipe->tmp_page) in free_pipe_info()
869 __free_page(pipe->tmp_page); in free_pipe_info()
870 kfree(pipe->bufs); in free_pipe_info()
871 kfree(pipe); in free_pipe_info()
881 return dynamic_dname(buffer, buflen, "pipe:[%lu]", in pipefs_dname()
892 struct pipe_inode_info *pipe; in get_pipe_inode() local
899 pipe = alloc_pipe_info(); in get_pipe_inode()
900 if (!pipe) in get_pipe_inode()
903 inode->i_pipe = pipe; in get_pipe_inode()
904 pipe->files = 2; in get_pipe_inode()
905 pipe->readers = pipe->writers = 1; in get_pipe_inode()
997 /* pipe groks IOCB_NOWAIT */ in __do_pipe_flags()
1023 * a pipe. It's not the way Unix traditionally does this, though.
1052 SYSCALL_DEFINE1(pipe, int __user *, fildes) in SYSCALL_DEFINE1() argument
1058 * This is the stupid "wait for pipe to be readable or writable"
1066 void pipe_wait_readable(struct pipe_inode_info *pipe) in pipe_wait_readable() argument
1068 pipe_unlock(pipe); in pipe_wait_readable()
1069 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); in pipe_wait_readable()
1070 pipe_lock(pipe); in pipe_wait_readable()
1073 void pipe_wait_writable(struct pipe_inode_info *pipe) in pipe_wait_writable() argument
1075 pipe_unlock(pipe); in pipe_wait_writable()
1076 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); in pipe_wait_writable()
1077 pipe_lock(pipe); in pipe_wait_writable()
1082 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1087 * because of the pipe lock, we can check the condition before being on
1090 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1092 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) in wait_for_partner() argument
1098 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); in wait_for_partner()
1099 pipe_unlock(pipe); in wait_for_partner()
1101 finish_wait(&pipe->rd_wait, &rdwait); in wait_for_partner()
1102 pipe_lock(pipe); in wait_for_partner()
1109 static void wake_up_partner(struct pipe_inode_info *pipe) in wake_up_partner() argument
1111 wake_up_interruptible_all(&pipe->rd_wait); in wake_up_partner()
1116 struct pipe_inode_info *pipe; in fifo_open() local
1124 pipe = inode->i_pipe; in fifo_open()
1125 pipe->files++; in fifo_open()
1129 pipe = alloc_pipe_info(); in fifo_open()
1130 if (!pipe) in fifo_open()
1132 pipe->files = 1; in fifo_open()
1137 free_pipe_info(pipe); in fifo_open()
1138 pipe = inode->i_pipe; in fifo_open()
1140 inode->i_pipe = pipe; in fifo_open()
1144 filp->private_data = pipe; in fifo_open()
1145 /* OK, we have a pipe and it's pinned down */ in fifo_open()
1147 __pipe_lock(pipe); in fifo_open()
1159 pipe->r_counter++; in fifo_open()
1160 if (pipe->readers++ == 0) in fifo_open()
1161 wake_up_partner(pipe); in fifo_open()
1163 if (!is_pipe && !pipe->writers) { in fifo_open()
1167 filp->f_version = pipe->w_counter; in fifo_open()
1169 if (wait_for_partner(pipe, &pipe->w_counter)) in fifo_open()
1182 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) in fifo_open()
1185 pipe->w_counter++; in fifo_open()
1186 if (!pipe->writers++) in fifo_open()
1187 wake_up_partner(pipe); in fifo_open()
1189 if (!is_pipe && !pipe->readers) { in fifo_open()
1190 if (wait_for_partner(pipe, &pipe->r_counter)) in fifo_open()
1203 pipe->readers++; in fifo_open()
1204 pipe->writers++; in fifo_open()
1205 pipe->r_counter++; in fifo_open()
1206 pipe->w_counter++; in fifo_open()
1207 if (pipe->readers == 1 || pipe->writers == 1) in fifo_open()
1208 wake_up_partner(pipe); in fifo_open()
1217 __pipe_unlock(pipe); in fifo_open()
1221 if (!--pipe->readers) in fifo_open()
1222 wake_up_interruptible(&pipe->wr_wait); in fifo_open()
1227 if (!--pipe->writers) in fifo_open()
1228 wake_up_interruptible_all(&pipe->rd_wait); in fifo_open()
1233 __pipe_unlock(pipe); in fifo_open()
1235 put_pipe_info(inode, pipe); in fifo_open()
1252 * Currently we rely on the pipe array holding a power-of-2 number
1260 /* Minimum pipe size, as required by POSIX */ in round_pipe_size()
1268 * Resize the pipe ring to a number of slots.
1270 * Note the pipe can be reduced in capacity, but only if the current
1274 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) in pipe_resize_ring() argument
1284 spin_lock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1285 mask = pipe->ring_size - 1; in pipe_resize_ring()
1286 head = pipe->head; in pipe_resize_ring()
1287 tail = pipe->tail; in pipe_resize_ring()
1291 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1297 * The pipe array wraps around, so just start the new one at zero in pipe_resize_ring()
1304 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1307 unsigned int tsize = pipe->ring_size - t; in pipe_resize_ring()
1309 memcpy(bufs + tsize, pipe->bufs, in pipe_resize_ring()
1311 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1319 kfree(pipe->bufs); in pipe_resize_ring()
1320 pipe->bufs = bufs; in pipe_resize_ring()
1321 pipe->ring_size = nr_slots; in pipe_resize_ring()
1322 if (pipe->max_usage > nr_slots) in pipe_resize_ring()
1323 pipe->max_usage = nr_slots; in pipe_resize_ring()
1324 pipe->tail = tail; in pipe_resize_ring()
1325 pipe->head = head; in pipe_resize_ring()
1327 if (!pipe_has_watch_queue(pipe)) { in pipe_resize_ring()
1328 pipe->max_usage = nr_slots; in pipe_resize_ring()
1329 pipe->nr_accounted = nr_slots; in pipe_resize_ring()
1332 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1335 wake_up_interruptible(&pipe->wr_wait); in pipe_resize_ring()
1340 * Allocate a new array of pipe buffers and copy the info over. Returns the
1341 * pipe size if successful, or return -ERROR on error.
1343 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg) in pipe_set_size() argument
1349 if (pipe_has_watch_queue(pipe)) in pipe_set_size()
1359 * If trying to increase the pipe capacity, check that an in pipe_set_size()
1362 * Decreasing the pipe capacity is always permitted, even in pipe_set_size()
1365 if (nr_slots > pipe->max_usage && in pipe_set_size()
1369 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); in pipe_set_size()
1371 if (nr_slots > pipe->max_usage && in pipe_set_size()
1379 ret = pipe_resize_ring(pipe, nr_slots); in pipe_set_size()
1383 return pipe->max_usage * PAGE_SIZE; in pipe_set_size()
1386 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); in pipe_set_size()
1392 * not enough to verify that this is a pipe.
1396 struct pipe_inode_info *pipe = file->private_data; in get_pipe_info() local
1398 if (file->f_op != &pipefifo_fops || !pipe) in get_pipe_info()
1400 if (for_splice && pipe_has_watch_queue(pipe)) in get_pipe_info()
1402 return pipe; in get_pipe_info()
1407 struct pipe_inode_info *pipe; in pipe_fcntl() local
1410 pipe = get_pipe_info(file, false); in pipe_fcntl()
1411 if (!pipe) in pipe_fcntl()
1414 __pipe_lock(pipe); in pipe_fcntl()
1418 ret = pipe_set_size(pipe, arg); in pipe_fcntl()
1421 ret = pipe->max_usage * PAGE_SIZE; in pipe_fcntl()
1428 __pipe_unlock(pipe); in pipe_fcntl()
1441 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1490 .procname = "pipe-max-size",
1497 .procname = "pipe-user-pages-hard",
1504 .procname = "pipe-user-pages-soft",