Lines Matching refs:epi
251 struct epitem *epi; member
308 struct epitem *epi= container_of(p, struct epitem, fllink); in unlist_file() local
309 spin_lock(&epi->ffd.file->f_lock); in unlist_file()
313 spin_unlock(&epi->ffd.file->f_lock); in unlist_file()
369 static inline int ep_is_linked(struct epitem *epi) in ep_is_linked() argument
371 return !list_empty(&epi->rdllink); in ep_is_linked()
475 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
477 struct eventpoll *ep = epi->ep; in ep_set_busy_poll_napi_id()
485 sock = sock_from_file(epi->ffd.file); in ep_set_busy_poll_napi_id()
572 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
619 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, in ep_poll_safewake() argument
641 if (epi) { in ep_poll_safewake()
642 if ((is_file_epoll(epi->ffd.file))) { in ep_poll_safewake()
643 ep_src = epi->ffd.file->private_data; in ep_poll_safewake()
658 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, in ep_poll_safewake() argument
687 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
689 struct eppoll_entry **p = &epi->pwqlist; in ep_unregister_pollwait()
700 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) in ep_wakeup_source() argument
702 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
706 static inline void ep_pm_stay_awake(struct epitem *epi) in ep_pm_stay_awake() argument
708 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_pm_stay_awake()
714 static inline bool ep_has_wakeup_source(struct epitem *epi) in ep_has_wakeup_source() argument
716 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
720 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) in ep_pm_stay_awake_rcu() argument
725 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
756 struct epitem *epi, *nepi; in ep_done_scan() local
764 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; in ep_done_scan()
765 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_done_scan()
772 if (!ep_is_linked(epi)) { in ep_done_scan()
777 list_add(&epi->rdllink, &ep->rdllist); in ep_done_scan()
778 ep_pm_stay_awake(epi); in ep_done_scan()
837 static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) in __ep_remove() argument
839 struct file *file = epi->ffd.file; in __ep_remove()
848 ep_unregister_pollwait(ep, epi); in __ep_remove()
852 if (epi->dying && !force) { in __ep_remove()
859 if (head->first == &epi->fllink && !epi->fllink.next) { in __ep_remove()
869 hlist_del_rcu(&epi->fllink); in __ep_remove()
873 rb_erase_cached(&epi->rbn, &ep->rbr); in __ep_remove()
876 if (ep_is_linked(epi)) in __ep_remove()
877 list_del_init(&epi->rdllink); in __ep_remove()
880 wakeup_source_unregister(ep_wakeup_source(epi)); in __ep_remove()
888 kfree_rcu(epi, rcu); in __ep_remove()
897 static void ep_remove_safe(struct eventpoll *ep, struct epitem *epi) in ep_remove_safe() argument
899 if (__ep_remove(ep, epi, false)) in ep_remove_safe()
906 struct epitem *epi; in ep_clear_and_put() local
918 epi = rb_entry(rbp, struct epitem, rbn); in ep_clear_and_put()
920 ep_unregister_pollwait(ep, epi); in ep_clear_and_put()
934 epi = rb_entry(rbp, struct epitem, rbn); in ep_clear_and_put()
935 ep_remove_safe(ep, epi); in ep_clear_and_put()
975 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth);
981 struct epitem *epi, *tmp; in __ep_eventpoll_poll() local
996 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { in __ep_eventpoll_poll()
997 if (ep_item_poll(epi, &pt, depth + 1)) { in __ep_eventpoll_poll()
1006 __pm_relax(ep_wakeup_source(epi)); in __ep_eventpoll_poll()
1007 list_del_init(&epi->rdllink); in __ep_eventpoll_poll()
1033 static struct file *epi_fget(const struct epitem *epi) in epi_fget() argument
1037 file = epi->ffd.file; in epi_fget()
1048 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, in ep_item_poll() argument
1051 struct file *file = epi_fget(epi); in ep_item_poll()
1061 pt->_key = epi->event.events; in ep_item_poll()
1067 return res & epi->event.events; in ep_item_poll()
1083 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); in ep_show_fdinfo() local
1084 struct inode *inode = file_inode(epi->ffd.file); in ep_show_fdinfo()
1088 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
1089 (long long)epi->event.data, in ep_show_fdinfo()
1090 (long long)epi->ffd.file->f_pos, in ep_show_fdinfo()
1119 struct epitem *epi; in eventpoll_release_file() local
1130 epi = hlist_entry(file->f_ep->first, struct epitem, fllink); in eventpoll_release_file()
1131 epi->dying = true; in eventpoll_release_file()
1138 ep = epi->ep; in eventpoll_release_file()
1140 dispose = __ep_remove(ep, epi, true); in eventpoll_release_file()
1182 struct epitem *epi, *epir = NULL; in ep_find() local
1187 epi = rb_entry(rbp, struct epitem, rbn); in ep_find()
1188 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
1194 epir = epi; in ep_find()
1206 struct epitem *epi; in ep_find_tfd() local
1209 epi = rb_entry(rbp, struct epitem, rbn); in ep_find_tfd()
1210 if (epi->ffd.fd == tfd) { in ep_find_tfd()
1212 return epi; in ep_find_tfd()
1227 struct epitem *epi; in get_epoll_tfile_raw_ptr() local
1235 epi = ep_find_tfd(ep, tfd, toff); in get_epoll_tfile_raw_ptr()
1236 if (epi) in get_epoll_tfile_raw_ptr()
1237 file_raw = epi->ffd.file; in get_epoll_tfile_raw_ptr()
1254 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback() local
1255 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1262 ep_set_busy_poll_napi_id(epi); in ep_poll_callback()
1270 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1279 if (pollflags && !(pollflags & epi->event.events)) in ep_poll_callback()
1289 if (epi->next == EP_UNACTIVE_PTR) { in ep_poll_callback()
1290 epi->next = READ_ONCE(ep->ovflist); in ep_poll_callback()
1291 WRITE_ONCE(ep->ovflist, epi); in ep_poll_callback()
1292 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1294 } else if (!ep_is_linked(epi)) { in ep_poll_callback()
1296 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_poll_callback()
1297 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1305 if ((epi->event.events & EPOLLEXCLUSIVE) && in ep_poll_callback()
1309 if (epi->event.events & EPOLLIN) in ep_poll_callback()
1313 if (epi->event.events & EPOLLOUT) in ep_poll_callback()
1334 ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); in ep_poll_callback()
1336 if (!(epi->event.events & EPOLLEXCLUSIVE)) in ep_poll_callback()
1366 struct epitem *epi = epq->epi; in ep_ptable_queue_proc() local
1369 if (unlikely(!epi)) // an earlier allocation has failed in ep_ptable_queue_proc()
1374 epq->epi = NULL; in ep_ptable_queue_proc()
1380 pwq->base = epi; in ep_ptable_queue_proc()
1381 if (epi->event.events & EPOLLEXCLUSIVE) in ep_ptable_queue_proc()
1385 pwq->next = epi->pwqlist; in ep_ptable_queue_proc()
1386 epi->pwqlist = pwq; in ep_ptable_queue_proc()
1389 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1399 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1406 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1407 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1449 struct epitem *epi; in reverse_path_check_proc() local
1455 hlist_for_each_entry_rcu(epi, refs, fllink) { in reverse_path_check_proc()
1456 struct hlist_head *refs = &epi->ep->refs; in reverse_path_check_proc()
1493 static int ep_create_wakeup_source(struct epitem *epi) in ep_create_wakeup_source() argument
1498 if (!epi->ep->ws) { in ep_create_wakeup_source()
1499 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1500 if (!epi->ep->ws) in ep_create_wakeup_source()
1504 take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry); in ep_create_wakeup_source()
1510 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1516 static noinline void ep_destroy_wakeup_source(struct epitem *epi) in ep_destroy_wakeup_source() argument
1518 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_destroy_wakeup_source()
1520 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1531 static int attach_epitem(struct file *file, struct epitem *epi) in attach_epitem() argument
1559 hlist_add_head_rcu(&epi->fllink, file->f_ep); in attach_epitem()
1573 struct epitem *epi; in ep_insert() local
1587 if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) { in ep_insert()
1593 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1594 epi->ep = ep; in ep_insert()
1595 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1596 epi->event = *event; in ep_insert()
1597 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1602 if (unlikely(attach_epitem(tfile, epi) < 0)) { in ep_insert()
1605 kmem_cache_free(epi_cache, epi); in ep_insert()
1617 ep_rbtree_insert(ep, epi); in ep_insert()
1629 ep_remove_safe(ep, epi); in ep_insert()
1633 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1634 error = ep_create_wakeup_source(epi); in ep_insert()
1636 ep_remove_safe(ep, epi); in ep_insert()
1642 epq.epi = epi; in ep_insert()
1652 revents = ep_item_poll(epi, &epq.pt, 1); in ep_insert()
1659 if (unlikely(!epq.epi)) { in ep_insert()
1660 ep_remove_safe(ep, epi); in ep_insert()
1668 ep_set_busy_poll_napi_id(epi); in ep_insert()
1671 if (revents && !ep_is_linked(epi)) { in ep_insert()
1672 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1673 ep_pm_stay_awake(epi); in ep_insert()
1695 static int ep_modify(struct eventpoll *ep, struct epitem *epi, in ep_modify() argument
1710 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1711 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1712 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1713 if (!ep_has_wakeup_source(epi)) in ep_modify()
1714 ep_create_wakeup_source(epi); in ep_modify()
1715 } else if (ep_has_wakeup_source(epi)) { in ep_modify()
1716 ep_destroy_wakeup_source(epi); in ep_modify()
1745 if (ep_item_poll(epi, &pt, 1)) { in ep_modify()
1747 if (!ep_is_linked(epi)) { in ep_modify()
1748 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1749 ep_pm_stay_awake(epi); in ep_modify()
1770 struct epitem *epi, *tmp; in ep_send_events() local
1792 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { in ep_send_events()
1808 ws = ep_wakeup_source(epi); in ep_send_events()
1815 list_del_init(&epi->rdllink); in ep_send_events()
1822 revents = ep_item_poll(epi, &pt, 1); in ep_send_events()
1826 events = epoll_put_uevent(revents, epi->event.data, events); in ep_send_events()
1828 list_add(&epi->rdllink, &txlist); in ep_send_events()
1829 ep_pm_stay_awake(epi); in ep_send_events()
1835 if (epi->event.events & EPOLLONESHOT) in ep_send_events()
1836 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events()
1837 else if (!(epi->event.events & EPOLLET)) { in ep_send_events()
1849 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events()
1850 ep_pm_stay_awake(epi); in ep_send_events()
2075 struct epitem *epi; in ep_loop_check_proc() local
2083 epi = rb_entry(rbp, struct epitem, rbn); in ep_loop_check_proc()
2084 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
2086 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
2102 list_file(epi->ffd.file); in ep_loop_check_proc()
2115 struct epitem *epi; in ep_get_upwards_depth_proc() local
2119 hlist_for_each_entry_rcu(epi, &ep->refs, fllink) in ep_get_upwards_depth_proc()
2120 result = max(result, ep_get_upwards_depth_proc(epi->ep, depth + 1) + 1); in ep_get_upwards_depth_proc()
2246 struct epitem *epi; in do_epoll_ctl() local
2338 epi = ep_find(ep, fd_file(tf), fd); in do_epoll_ctl()
2343 if (!epi) { in do_epoll_ctl()
2350 if (epi) { in do_epoll_ctl()
2355 ep_remove_safe(ep, epi); in do_epoll_ctl()
2362 if (epi) { in do_epoll_ctl()
2363 if (!(epi->event.events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2365 error = ep_modify(ep, epi, epds); in do_epoll_ctl()