linux/fs/eventpoll.c

13 #include <linux/file.h>
63  * insertion does not create a cycle of epoll file descriptors, which
72  * before e2->mtx). Since we disallow cycles of epoll file
75  * of epoll file descriptors, we use the current recursion depth as
103 	struct file *file;
116 	 * Wait queue item that will be linked to the target file wait
126  * Each file descriptor added to the eventpoll interface will
148 	/* The file descriptor information this item refers to */
152 	 * Protected by file->f_lock, true for to-be-released epitem already
153 	 * removed from the "struct file" items list; together with
164 	/* List header used to link this item to the "struct file" items list */
175  * This structure is stored inside the "private_data" member of the file
183 	 * collection loop, the file cleanup path, the epoll file exit
191 	/* Wait queue used by file->poll() */
194 	/* List of ready file descriptors */
216 	struct file *file;
262 /* Used to check for epoll file descriptor inclusion loops */
289 static void list_file(struct file *file)
293 	head = container_of(file->f_ep, struct epitems_head, epitems);
306 		spin_lock(&epi->ffd.file->f_lock);
310 		spin_unlock(&epi->ffd.file->f_lock);
344 static inline int is_file_epoll(struct file *f)
351 			      struct file *file, int fd)
353 	ffd->file = file;
361 	return (p1->file > p2->file ? +1:
362 	        (p1->file < p2->file ? -1 : p1->fd - p2->fd));
482 	sock = sock_from_file(epi->ffd.file);
503 static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,
506 	struct eventpoll *ep = file->private_data;
573 static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,
632 	 * not an epoll file itself, we are at depth 1 since the wakeup source
633 	 * is depth 0. If the wakeup source is a previous epoll file in the
635 	 * nests + 1. The previous epoll file nests value is stable since its
639 		if ((is_file_epoll(epi->ffd.file))) {
640 			ep_src = epi->ffd.file->private_data;
681  * This function unregisters poll callbacks from the associated file
835 	struct file *file = epi->ffd.file;
847 	spin_lock(&file->f_lock);
849 		spin_unlock(&file->f_lock);
854 	head = file->f_ep;
857 		WRITE_ONCE(file->f_ep, NULL);
858 		if (!is_file_epoll(file)) {
866 	spin_unlock(&file->f_lock);
904 	/* We need to release all tasks waiting for these file */
940 static long ep_eventpoll_ioctl(struct file *file, unsigned int cmd,
945 	if (!is_file_epoll(file))
951 		ret = ep_eventpoll_bp_ioctl(file, cmd, arg);
961 static int ep_eventpoll_release(struct inode *inode, struct file *file)
963 	struct eventpoll *ep = file->private_data;
973 static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth)
975 	struct eventpoll *ep = file->private_data;
984 	poll_wait(file, &ep->poll_wait, wait);
1012  * The ffd.file pointer may be in the process of being torn down due to
1015  * Normally, even with the atomic_long_inc_not_zero, the file may have
1019  * But for epoll, users hold the ep->mtx mutex, and as such any file in
1021  * and thus the underlying file allocation will not be free'd, and the
1022  * file re-use cannot happen.
1025  * operation - 'ffd.file' cannot go away even if the refcount has
1029 static struct file *epi_fget(const struct epitem *epi)
1031 	struct file *file;
1033 	file = epi->ffd.file;
1034 	if (!file_ref_get(&file->f_ref))
1035 		file = NULL;
1036 	return file;
1047 	struct file *file = epi_fget(epi);
1052 	 * treat this more as "file doesn't exist, poll didn't happen".
1054 	if (!file)
1058 	if (!is_file_epoll(file))
1059 		res = vfs_poll(file, pt);
1061 		res = __ep_eventpoll_poll(file, pt, depth);
1062 	fput(file);
1066 static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
1068 	return __ep_eventpoll_poll(file, wait, 0);
1072 static void ep_show_fdinfo(struct seq_file *m, struct file *f)
1080 		struct inode *inode = file_inode(epi->ffd.file);
1086 			   (long long)epi->ffd.file->f_pos,
1095 /* File callbacks that implement the eventpoll file behaviour */
1112 void eventpoll_release_file(struct file *file)
1124 	spin_lock(&file->f_lock);
1125 	if (file->f_ep && file->f_ep->first) {
1126 		epi = hlist_entry(file->f_ep->first, struct epitem, fllink);
1128 		spin_unlock(&file->f_lock);
1143 	spin_unlock(&file->f_lock);
1170  * Search the file inside the eventpoll tree. The RB tree operations
1174 static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
1181 	ep_set_ffd(&ffd, file, fd);
1218 struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
1221 	struct file *file_raw;
1225 	if (!is_file_epoll(file))
1228 	ep = file->private_data;
1233 		file_raw = epi->ffd.file;
1321  * mechanism. It is called by the stored file descriptors when they
1325  * events from another file descriptor, thus all modifications to ->rdllist
1443  * target file wakeup lists.
1445 static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
1498  * from a single file of interest. For example, we allow 1000 paths of length
1499  * 1, to emanate from each file of interest. This essentially represents the
1502  * is connected to n file sources. In this case each file source has 1 path
1587 	take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);
1614 static int attach_epitem(struct file *file, struct epitem *epi)
1620 	if (is_file_epoll(file))
1621 		ep = file->private_data;
1625 	} else if (!READ_ONCE(file->f_ep)) {
1632 	spin_lock(&file->f_lock);
1633 	if (!file->f_ep) {
1635 			spin_unlock(&file->f_lock);
1639 		WRITE_ONCE(file->f_ep, head);
1642 	hlist_add_head_rcu(&epi->fllink, file->f_ep);
1643 	spin_unlock(&file->f_lock);
1652 		     struct file *tfile, int fd, int full_check)
1684 	/* Add the current item to the list of active epoll hook for this file */
1706 	 * ep_free() as the ep file itself still holds an ep reference.
1730 	 * We can safely use the file* here because its usage count has
1753 	/* If the file is already "ready" we drop it inside the ready list */
1776  * has a match in the current file status. Must be called with "mtx" held.
1823 	 * Get current event bits. We can safely use the file* here because
1922 			 * If this file has been added with Level
2144  * ep_loop_check_proc - verify that adding an epoll file @ep inside another
2145  *                      epoll file does not create closed loops, and
2166 		if (unlikely(is_file_epoll(epi->ffd.file))) {
2168 			ep_tovisit = epi->ffd.file->private_data;
2177 			 * If we've reached a file that is not associated with
2184 			list_file(epi->ffd.file);
2209  * ep_loop_check - Performs a check to verify that adding an epoll file (@to)
2210  *                 into another epoll file (represented by @ep) does not create
2251  * Open an eventpoll file descriptor.
2257 	struct file *file;
2271 	 * Creates all the items needed to setup an eventpoll file. That is,
2272 	 * a file structure and a free file descriptor.
2279 	file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
2281 	if (IS_ERR(file)) {
2282 		error = PTR_ERR(file);
2285 	ep->file = file;
2286 	fd_install(fd, file);
2347 	/* Get the "struct file *" for the target file */
2352 	/* The target file descriptor must support poll */
2361 	 * We have to check that the file structure underneath the file descriptor
2362 	 * the user passed to us _is_ an eventpoll file. And also we do not permit
2363 	 * adding an epoll file descriptor inside itself.
2389 	 * When we insert an epoll file descriptor inside another epoll file
2397 	 * the epoll file descriptor is attaching directly to a wakeup source,
2398 	 * unless the epoll file descriptor is nested. The purpose of taking the
2428 	 * Try to lookup the file inside our RB tree. Since we grabbed "mtx"
2478  * the eventpoll file that enables the insertion/removal/change of
2479  * file descriptors inside the interest set.
2493 static int ep_check_params(struct file *file, struct epoll_event __user *evs,
2505 	 * We have to check that the file structure underneath the fd
2506 	 * the user passed to us _is_ an eventpoll file.
2508 	if (!is_file_epoll(file))
2514 int epoll_sendevents(struct file *file, struct epoll_event __user *events,
2520 	ret = ep_check_params(file, events, maxevents);
2524 	ep = file->private_data;
2535  * Implement the event wait interface for the eventpoll file. It is the kernel
2544 	/* Get the "struct file *" for the eventpoll file */
2573  * Implement the event wait interface for the eventpoll file. It is the kernel