Lines Matching full:pipe
3 * linux/fs/pipe.c
35 * The max size that a non-root user is allowed to grow the pipe. Can
36 * be set by root in /proc/sys/fs/pipe-max-size
63 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass) in pipe_lock_nested() argument
65 if (pipe->files) in pipe_lock_nested()
66 mutex_lock_nested(&pipe->mutex, subclass); in pipe_lock_nested()
69 void pipe_lock(struct pipe_inode_info *pipe) in pipe_lock() argument
72 * pipe_lock() nests non-pipe inode locks (for writing to a file) in pipe_lock()
74 pipe_lock_nested(pipe, I_MUTEX_PARENT); in pipe_lock()
78 void pipe_unlock(struct pipe_inode_info *pipe) in pipe_unlock() argument
80 if (pipe->files) in pipe_unlock()
81 mutex_unlock(&pipe->mutex); in pipe_unlock()
85 static inline void __pipe_lock(struct pipe_inode_info *pipe) in __pipe_lock() argument
87 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT); in __pipe_lock()
90 static inline void __pipe_unlock(struct pipe_inode_info *pipe) in __pipe_unlock() argument
92 mutex_unlock(&pipe->mutex); in __pipe_unlock()
109 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, in anon_pipe_buf_release() argument
119 if (page_count(page) == 1 && !pipe->tmp_page) in anon_pipe_buf_release()
120 pipe->tmp_page = page; in anon_pipe_buf_release()
125 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, in anon_pipe_buf_try_steal() argument
139 * @pipe: the pipe that the buffer belongs to
149 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, in generic_pipe_buf_try_steal() argument
169 * @pipe: the pipe that the buffer belongs to
175 * pipe into another.
177 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in generic_pipe_buf_get() argument
185 * @pipe: the pipe that the buffer belongs to
191 void generic_pipe_buf_release(struct pipe_inode_info *pipe, in generic_pipe_buf_release() argument
204 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
205 static inline bool pipe_readable(const struct pipe_inode_info *pipe) in pipe_readable() argument
207 unsigned int head = READ_ONCE(pipe->head); in pipe_readable()
208 unsigned int tail = READ_ONCE(pipe->tail); in pipe_readable()
209 unsigned int writers = READ_ONCE(pipe->writers); in pipe_readable()
219 struct pipe_inode_info *pipe = filp->private_data; in pipe_read() local
228 __pipe_lock(pipe); in pipe_read()
231 * We only wake up writers if the pipe was full when we started in pipe_read()
238 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
240 unsigned int head = pipe->head; in pipe_read()
241 unsigned int tail = pipe->tail; in pipe_read()
242 unsigned int mask = pipe->ring_size - 1; in pipe_read()
245 if (pipe->note_loss) { in pipe_read()
264 pipe->note_loss = false; in pipe_read()
269 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in pipe_read()
283 error = pipe_buf_confirm(pipe, buf); in pipe_read()
307 pipe_buf_release(pipe, buf); in pipe_read()
308 spin_lock_irq(&pipe->rd_wait.lock); in pipe_read()
311 pipe->note_loss = true; in pipe_read()
314 pipe->tail = tail; in pipe_read()
315 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_read()
324 if (!pipe->writers) in pipe_read()
332 __pipe_unlock(pipe); in pipe_read()
338 * pipe buffer, and might have made space in the buffers in pipe_read()
341 * You can't make zero-sized pipe buffers by doing an empty in pipe_read()
348 * _very_ unlikely case that the pipe was full, but we got in pipe_read()
352 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
353 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
362 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) in pipe_read()
365 __pipe_lock(pipe); in pipe_read()
366 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
369 if (pipe_empty(pipe->head, pipe->tail)) in pipe_read()
371 __pipe_unlock(pipe); in pipe_read()
374 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
375 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
378 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_read()
389 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
390 static inline bool pipe_writable(const struct pipe_inode_info *pipe) in pipe_writable() argument
392 unsigned int head = READ_ONCE(pipe->head); in pipe_writable()
393 unsigned int tail = READ_ONCE(pipe->tail); in pipe_writable()
394 unsigned int max_usage = READ_ONCE(pipe->max_usage); in pipe_writable()
397 !READ_ONCE(pipe->readers); in pipe_writable()
404 struct pipe_inode_info *pipe = filp->private_data; in pipe_write() local
416 __pipe_lock(pipe); in pipe_write()
418 if (!pipe->readers) { in pipe_write()
425 if (pipe->watch_queue) { in pipe_write()
432 * Only wake up if the pipe started out empty, since in pipe_write()
442 head = pipe->head; in pipe_write()
443 was_empty = pipe_empty(head, pipe->tail); in pipe_write()
446 unsigned int mask = pipe->ring_size - 1; in pipe_write()
447 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask]; in pipe_write()
452 ret = pipe_buf_confirm(pipe, buf); in pipe_write()
469 if (!pipe->readers) { in pipe_write()
476 head = pipe->head; in pipe_write()
477 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { in pipe_write()
478 unsigned int mask = pipe->ring_size - 1; in pipe_write()
479 struct pipe_buffer *buf = &pipe->bufs[head & mask]; in pipe_write()
480 struct page *page = pipe->tmp_page; in pipe_write()
489 pipe->tmp_page = page; in pipe_write()
497 spin_lock_irq(&pipe->rd_wait.lock); in pipe_write()
499 head = pipe->head; in pipe_write()
500 if (pipe_full(head, pipe->tail, pipe->max_usage)) { in pipe_write()
501 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_write()
505 pipe->head = head + 1; in pipe_write()
506 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_write()
509 buf = &pipe->bufs[head & mask]; in pipe_write()
518 pipe->tmp_page = NULL; in pipe_write()
534 if (!pipe_full(head, pipe->tail, pipe->max_usage)) in pipe_write()
550 * We're going to release the pipe lock and wait for more in pipe_write()
552 * after waiting we need to re-check whether the pipe in pipe_write()
555 __pipe_unlock(pipe); in pipe_write()
557 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
558 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
560 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); in pipe_write()
561 __pipe_lock(pipe); in pipe_write()
562 was_empty = pipe_empty(pipe->head, pipe->tail); in pipe_write()
566 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in pipe_write()
568 __pipe_unlock(pipe); in pipe_write()
580 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
581 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
584 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_write()
596 struct pipe_inode_info *pipe = filp->private_data; in pipe_ioctl() local
601 __pipe_lock(pipe); in pipe_ioctl()
603 head = pipe->head; in pipe_ioctl()
604 tail = pipe->tail; in pipe_ioctl()
605 mask = pipe->ring_size - 1; in pipe_ioctl()
608 count += pipe->bufs[tail & mask].len; in pipe_ioctl()
611 __pipe_unlock(pipe); in pipe_ioctl()
618 __pipe_lock(pipe); in pipe_ioctl()
619 ret = watch_queue_set_size(pipe, arg); in pipe_ioctl()
620 __pipe_unlock(pipe); in pipe_ioctl()
626 pipe, (struct watch_notification_filter __user *)arg); in pipe_ioctl()
639 struct pipe_inode_info *pipe = filp->private_data; in pipe_poll() local
643 * Reading pipe state only -- no need for acquiring the semaphore. in pipe_poll()
649 poll_wait(filp, &pipe->rd_wait, wait); in pipe_poll()
651 poll_wait(filp, &pipe->wr_wait, wait); in pipe_poll()
658 head = READ_ONCE(pipe->head); in pipe_poll()
659 tail = READ_ONCE(pipe->tail); in pipe_poll()
665 if (!pipe->writers && filp->f_version != pipe->w_counter) in pipe_poll()
670 if (!pipe_full(head, tail, pipe->max_usage)) in pipe_poll()
676 if (!pipe->readers) in pipe_poll()
683 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) in put_pipe_info() argument
688 if (!--pipe->files) { in put_pipe_info()
695 free_pipe_info(pipe); in put_pipe_info()
701 struct pipe_inode_info *pipe = file->private_data; in pipe_release() local
703 __pipe_lock(pipe); in pipe_release()
705 pipe->readers--; in pipe_release()
707 pipe->writers--; in pipe_release()
710 if (!pipe->readers != !pipe->writers) { in pipe_release()
711 wake_up_interruptible_all(&pipe->rd_wait); in pipe_release()
712 wake_up_interruptible_all(&pipe->wr_wait); in pipe_release()
713 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_release()
714 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_release()
716 __pipe_unlock(pipe); in pipe_release()
718 put_pipe_info(inode, pipe); in pipe_release()
725 struct pipe_inode_info *pipe = filp->private_data; in pipe_fasync() local
728 __pipe_lock(pipe); in pipe_fasync()
730 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); in pipe_fasync()
732 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); in pipe_fasync()
735 fasync_helper(-1, filp, 0, &pipe->fasync_readers); in pipe_fasync()
737 __pipe_unlock(pipe); in pipe_fasync()
768 struct pipe_inode_info *pipe; in alloc_pipe_info() local
774 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT); in alloc_pipe_info()
775 if (pipe == NULL) in alloc_pipe_info()
791 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), in alloc_pipe_info()
794 if (pipe->bufs) { in alloc_pipe_info()
795 init_waitqueue_head(&pipe->rd_wait); in alloc_pipe_info()
796 init_waitqueue_head(&pipe->wr_wait); in alloc_pipe_info()
797 pipe->r_counter = pipe->w_counter = 1; in alloc_pipe_info()
798 pipe->max_usage = pipe_bufs; in alloc_pipe_info()
799 pipe->ring_size = pipe_bufs; in alloc_pipe_info()
800 pipe->nr_accounted = pipe_bufs; in alloc_pipe_info()
801 pipe->user = user; in alloc_pipe_info()
802 mutex_init(&pipe->mutex); in alloc_pipe_info()
803 return pipe; in alloc_pipe_info()
808 kfree(pipe); in alloc_pipe_info()
814 void free_pipe_info(struct pipe_inode_info *pipe) in free_pipe_info() argument
819 if (pipe->watch_queue) { in free_pipe_info()
820 watch_queue_clear(pipe->watch_queue); in free_pipe_info()
821 put_watch_queue(pipe->watch_queue); in free_pipe_info()
825 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); in free_pipe_info()
826 free_uid(pipe->user); in free_pipe_info()
827 for (i = 0; i < pipe->ring_size; i++) { in free_pipe_info()
828 struct pipe_buffer *buf = pipe->bufs + i; in free_pipe_info()
830 pipe_buf_release(pipe, buf); in free_pipe_info()
832 if (pipe->tmp_page) in free_pipe_info()
833 __free_page(pipe->tmp_page); in free_pipe_info()
834 kfree(pipe->bufs); in free_pipe_info()
835 kfree(pipe); in free_pipe_info()
845 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]", in pipefs_dname()
856 struct pipe_inode_info *pipe; in get_pipe_inode() local
863 pipe = alloc_pipe_info(); in get_pipe_inode()
864 if (!pipe) in get_pipe_inode()
867 inode->i_pipe = pipe; in get_pipe_inode()
868 pipe->files = 2; in get_pipe_inode()
869 pipe->readers = pipe->writers = 1; in get_pipe_inode()
984 * a pipe. It's not the way Unix traditionally does this, though.
1013 SYSCALL_DEFINE1(pipe, int __user *, fildes) in SYSCALL_DEFINE1() argument
1019 * This is the stupid "wait for pipe to be readable or writable"
1027 void pipe_wait_readable(struct pipe_inode_info *pipe) in pipe_wait_readable() argument
1029 pipe_unlock(pipe); in pipe_wait_readable()
1030 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); in pipe_wait_readable()
1031 pipe_lock(pipe); in pipe_wait_readable()
1034 void pipe_wait_writable(struct pipe_inode_info *pipe) in pipe_wait_writable() argument
1036 pipe_unlock(pipe); in pipe_wait_writable()
1037 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); in pipe_wait_writable()
1038 pipe_lock(pipe); in pipe_wait_writable()
1043 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1048 * because of the pipe lock, we can check the condition before being on
1051 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1053 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) in wait_for_partner() argument
1059 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); in wait_for_partner()
1060 pipe_unlock(pipe); in wait_for_partner()
1062 finish_wait(&pipe->rd_wait, &rdwait); in wait_for_partner()
1063 pipe_lock(pipe); in wait_for_partner()
1070 static void wake_up_partner(struct pipe_inode_info *pipe) in wake_up_partner() argument
1072 wake_up_interruptible_all(&pipe->rd_wait); in wake_up_partner()
1077 struct pipe_inode_info *pipe; in fifo_open() local
1085 pipe = inode->i_pipe; in fifo_open()
1086 pipe->files++; in fifo_open()
1090 pipe = alloc_pipe_info(); in fifo_open()
1091 if (!pipe) in fifo_open()
1093 pipe->files = 1; in fifo_open()
1098 free_pipe_info(pipe); in fifo_open()
1099 pipe = inode->i_pipe; in fifo_open()
1101 inode->i_pipe = pipe; in fifo_open()
1105 filp->private_data = pipe; in fifo_open()
1106 /* OK, we have a pipe and it's pinned down */ in fifo_open()
1108 __pipe_lock(pipe); in fifo_open()
1120 pipe->r_counter++; in fifo_open()
1121 if (pipe->readers++ == 0) in fifo_open()
1122 wake_up_partner(pipe); in fifo_open()
1124 if (!is_pipe && !pipe->writers) { in fifo_open()
1128 filp->f_version = pipe->w_counter; in fifo_open()
1130 if (wait_for_partner(pipe, &pipe->w_counter)) in fifo_open()
1143 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) in fifo_open()
1146 pipe->w_counter++; in fifo_open()
1147 if (!pipe->writers++) in fifo_open()
1148 wake_up_partner(pipe); in fifo_open()
1150 if (!is_pipe && !pipe->readers) { in fifo_open()
1151 if (wait_for_partner(pipe, &pipe->r_counter)) in fifo_open()
1164 pipe->readers++; in fifo_open()
1165 pipe->writers++; in fifo_open()
1166 pipe->r_counter++; in fifo_open()
1167 pipe->w_counter++; in fifo_open()
1168 if (pipe->readers == 1 || pipe->writers == 1) in fifo_open()
1169 wake_up_partner(pipe); in fifo_open()
1178 __pipe_unlock(pipe); in fifo_open()
1182 if (!--pipe->readers) in fifo_open()
1183 wake_up_interruptible(&pipe->wr_wait); in fifo_open()
1188 if (!--pipe->writers) in fifo_open()
1189 wake_up_interruptible_all(&pipe->rd_wait); in fifo_open()
1194 __pipe_unlock(pipe); in fifo_open()
1196 put_pipe_info(inode, pipe); in fifo_open()
1212 * Currently we rely on the pipe array holding a power-of-2 number
1220 /* Minimum pipe size, as required by POSIX */ in round_pipe_size()
1228 * Resize the pipe ring to a number of slots.
1230 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) in pipe_resize_ring() argument
1236 * We can shrink the pipe, if arg is greater than the ring occupancy. in pipe_resize_ring()
1238 * allocate again like we would do for growing. If the pipe currently in pipe_resize_ring()
1241 mask = pipe->ring_size - 1; in pipe_resize_ring()
1242 head = pipe->head; in pipe_resize_ring()
1243 tail = pipe->tail; in pipe_resize_ring()
1244 n = pipe_occupancy(pipe->head, pipe->tail); in pipe_resize_ring()
1254 * The pipe array wraps around, so just start the new one at zero in pipe_resize_ring()
1261 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1264 unsigned int tsize = pipe->ring_size - t; in pipe_resize_ring()
1266 memcpy(bufs + tsize, pipe->bufs, in pipe_resize_ring()
1268 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1276 kfree(pipe->bufs); in pipe_resize_ring()
1277 pipe->bufs = bufs; in pipe_resize_ring()
1278 pipe->ring_size = nr_slots; in pipe_resize_ring()
1279 if (pipe->max_usage > nr_slots) in pipe_resize_ring()
1280 pipe->max_usage = nr_slots; in pipe_resize_ring()
1281 pipe->tail = tail; in pipe_resize_ring()
1282 pipe->head = head; in pipe_resize_ring()
1285 wake_up_interruptible(&pipe->wr_wait); in pipe_resize_ring()
1290 * Allocate a new array of pipe buffers and copy the info over. Returns the
1291 * pipe size if successful, or return -ERROR on error.
1293 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) in pipe_set_size() argument
1300 if (pipe->watch_queue) in pipe_set_size()
1311 * If trying to increase the pipe capacity, check that an in pipe_set_size()
1314 * Decreasing the pipe capacity is always permitted, even in pipe_set_size()
1317 if (nr_slots > pipe->max_usage && in pipe_set_size()
1321 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); in pipe_set_size()
1323 if (nr_slots > pipe->max_usage && in pipe_set_size()
1331 ret = pipe_resize_ring(pipe, nr_slots); in pipe_set_size()
1335 pipe->max_usage = nr_slots; in pipe_set_size()
1336 pipe->nr_accounted = nr_slots; in pipe_set_size()
1337 return pipe->max_usage * PAGE_SIZE; in pipe_set_size()
1340 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); in pipe_set_size()
1347 * pipe.
1351 struct pipe_inode_info *pipe = file->private_data; in get_pipe_info() local
1353 if (file->f_op != &pipefifo_fops || !pipe) in get_pipe_info()
1356 if (for_splice && pipe->watch_queue) in get_pipe_info()
1359 return pipe; in get_pipe_info()
1364 struct pipe_inode_info *pipe; in pipe_fcntl() local
1367 pipe = get_pipe_info(file, false); in pipe_fcntl()
1368 if (!pipe) in pipe_fcntl()
1371 __pipe_lock(pipe); in pipe_fcntl()
1375 ret = pipe_set_size(pipe, arg); in pipe_fcntl()
1378 ret = pipe->max_usage * PAGE_SIZE; in pipe_fcntl()
1385 __pipe_unlock(pipe); in pipe_fcntl()
1398 * d_name - pipe: will go nicely and kill the special-casing in procfs.