Home
last modified time | relevance | path

Searched refs:wake_q (Results 1 – 20 of 20) sorted by relevance

/linux/include/linux/sched/
H A Dwake_q.h68 void raw_spin_unlock_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) in raw_spin_unlock_wake() argument
72 if (wake_q) { in raw_spin_unlock_wake()
73 wake_up_q(wake_q); in raw_spin_unlock_wake()
74 wake_q_init(wake_q); in raw_spin_unlock_wake()
79 void raw_spin_unlock_irq_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) in raw_spin_unlock_irq_wake() argument
83 if (wake_q) { in raw_spin_unlock_irq_wake()
84 wake_up_q(wake_q); in raw_spin_unlock_irq_wake()
85 wake_q_init(wake_q); in raw_spin_unlock_irq_wake()
91 struct wake_q_head *wake_q) in raw_spin_unlock_irqrestore_wake() argument
95 if (wake_q) { in raw_spin_unlock_irqrestore_wake()
[all …]
/linux/kernel/locking/
H A Dww_mutex.h278 struct ww_acquire_ctx *ww_ctx, struct wake_q_head *wake_q) in __ww_mutex_die() argument
293 wake_q_add(wake_q, waiter->task); in __ww_mutex_die()
309 struct wake_q_head *wake_q) in __ww_mutex_wound() argument
351 wake_q_add(wake_q, owner); in __ww_mutex_wound()
373 struct wake_q_head *wake_q) in __ww_mutex_check_waiters() argument
385 if (__ww_mutex_die(lock, cur, ww_ctx, wake_q) || in __ww_mutex_check_waiters()
386 __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx, wake_q)) in __ww_mutex_check_waiters()
398 DEFINE_WAKE_Q(wake_q); in ww_mutex_set_context_fastpath()
429 __ww_mutex_check_waiters(&lock->base, ctx, &wake_q); in ww_mutex_set_context_fastpath()
432 wake_up_q(&wake_q); in ww_mutex_set_context_fastpath()
[all …]
H A Drwsem.c412 struct wake_q_head *wake_q) in rwsem_mark_wake() argument
435 wake_q_add(wake_q, waiter->task); in rwsem_mark_wake()
565 wake_q_add_safe(wake_q, tsk); in rwsem_mark_wake()
577 struct wake_q_head *wake_q) in rwsem_del_wake_waiter() argument
582 wake_q_init(wake_q); in rwsem_del_wake_waiter()
590 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, wake_q); in rwsem_del_wake_waiter()
592 if (!wake_q_empty(wake_q)) in rwsem_del_wake_waiter()
593 wake_up_q(wake_q); in rwsem_del_wake_waiter()
973 struct wake_q_head *wake_q) in rwsem_cond_wake_waiter() argument
986 rwsem_mark_wake(sem, wake_type, wake_q); in rwsem_cond_wake_waiter()
[all …]
H A Drtmutex.c39 struct wake_q_head *wake_q) in __ww_mutex_add_waiter() argument
46 struct wake_q_head *wake_q) in __ww_mutex_check_waiters() argument
1208 struct wake_q_head *wake_q) in task_blocks_on_rt_mutex() argument
1252 res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q); in task_blocks_on_rt_mutex()
1296 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in task_blocks_on_rt_mutex()
1609 struct wake_q_head *wake_q) in rt_mutex_slowlock_block() argument
1643 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in rt_mutex_slowlock_block()
1696 struct wake_q_head *wake_q) in __rt_mutex_slowlock() argument
1708 __ww_mutex_check_waiters(rtm, ww_ctx, wake_q); in __rt_mutex_slowlock()
1719 ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk, wake_q); in __rt_mutex_slowlock()
[all …]
H A Dsemaphore.c43 static noinline void __up(struct semaphore *sem, struct wake_q_head *wake_q);
223 DEFINE_WAKE_Q(wake_q); in up()
232 __up(sem, &wake_q); in up()
234 if (!wake_q_empty(&wake_q)) in up()
235 wake_up_q(&wake_q); in up()
322 struct wake_q_head *wake_q) in __up() argument
328 wake_q_add(wake_q, waiter->task); in __up()
H A Dmutex.c566 DEFINE_WAKE_Q(wake_q); in __mutex_lock_common()
618 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); in __mutex_lock_common()
638 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q); in __mutex_lock_common()
674 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
723 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); in __mutex_lock_common()
738 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
749 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
918 DEFINE_WAKE_Q(wake_q); in __mutex_unlock_slowpath()
959 wake_q_add(&wake_q, next); in __mutex_unlock_slowpath()
965 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_unlock_slowpath()
H A Drtmutex_api.c314 struct wake_q_head *wake_q) in __rt_mutex_start_proxy_lock() argument
325 RT_MUTEX_FULL_CHAINWALK, wake_q); in __rt_mutex_start_proxy_lock()
364 DEFINE_WAKE_Q(wake_q); in rt_mutex_start_proxy_lock()
367 ret = __rt_mutex_start_proxy_lock(lock, waiter, task, &wake_q); in rt_mutex_start_proxy_lock()
372 wake_up_q(&wake_q); in rt_mutex_start_proxy_lock()
H A Drwbase_rt.c72 DEFINE_WAKE_Q(wake_q); in __rwbase_read_lock()
114 ret = rwbase_rtmutex_slowlock_locked(rtm, state, &wake_q); in __rwbase_read_lock()
128 wake_up_q(&wake_q); in __rwbase_read_lock()
H A Dspinlock_rt.c166 struct wake_q_head *wake_q) in rwbase_rtmutex_slowlock_locked() argument
168 rtlock_slowlock_locked(rtm, wake_q); in rwbase_rtmutex_slowlock_locked()
/linux/ipc/
H A Dsem.c787 struct wake_q_head *wake_q) in wake_up_sem_queue_prepare() argument
796 wake_q_add_safe(wake_q, sleeper); in wake_up_sem_queue_prepare()
855 struct wake_q_head *wake_q) in wake_const_ops() argument
874 wake_up_sem_queue_prepare(q, error, wake_q); in wake_const_ops()
894 int nsops, struct wake_q_head *wake_q) in do_smart_wakeup_zero() argument
907 semop_completed |= wake_const_ops(sma, num, wake_q); in do_smart_wakeup_zero()
918 semop_completed |= wake_const_ops(sma, i, wake_q); in do_smart_wakeup_zero()
927 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
949 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q) in update_queue() argument
986 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
[all …]
H A Dmsg.c213 struct wake_q_head *wake_q, bool kill) in ss_wakeup() argument
245 wake_q_add(wake_q, mss->tsk); in ss_wakeup()
250 struct wake_q_head *wake_q) in expunge_all() argument
261 wake_q_add_safe(wake_q, r_tsk); in expunge_all()
279 DEFINE_WAKE_Q(wake_q); in freeque()
281 expunge_all(msq, -EIDRM, &wake_q); in freeque()
282 ss_wakeup(msq, &wake_q, true); in freeque()
285 wake_up_q(&wake_q); in freeque()
432 DEFINE_WAKE_Q(wake_q); in msgctl_down()
452 expunge_all(msq, -EAGAIN, &wake_q); in msgctl_down()
[all …]
H A Dmqueue.c1018 static inline void __pipelined_op(struct wake_q_head *wake_q, in __pipelined_op() argument
1029 wake_q_add_safe(wake_q, task); in __pipelined_op()
1035 static inline void pipelined_send(struct wake_q_head *wake_q, in pipelined_send() argument
1041 __pipelined_op(wake_q, info, receiver); in pipelined_send()
1046 static inline void pipelined_receive(struct wake_q_head *wake_q, in pipelined_receive() argument
1059 __pipelined_op(wake_q, info, sender); in pipelined_receive()
1074 DEFINE_WAKE_Q(wake_q); in do_mq_timedsend()
1148 pipelined_send(&wake_q, info, msg_ptr, receiver); in do_mq_timedsend()
1160 wake_up_q(&wake_q); in do_mq_timedsend()
1234 DEFINE_WAKE_Q(wake_q); in do_mq_timedreceive()
[all …]
/linux/kernel/futex/
H A Dwaitwake.c134 void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) in futex_wake_mark() argument
149 wake_q_add_safe(wake_q, p); in futex_wake_mark()
159 DEFINE_WAKE_Q(wake_q); in futex_wake()
191 this->wake(&wake_q, this); in futex_wake()
198 wake_up_q(&wake_q); in futex_wake()
257 DEFINE_WAKE_Q(wake_q); in futex_wake_op()
305 this->wake(&wake_q, this); in futex_wake_op()
319 this->wake(&wake_q, this); in futex_wake_op()
330 wake_up_q(&wake_q); in futex_wake_op()
H A Drequeue.c387 DEFINE_WAKE_Q(wake_q); in futex_requeue()
610 this->wake(&wake_q, this); in futex_requeue()
692 wake_up_q(&wake_q); in futex_requeue()
H A Dpi.c924 DEFINE_WAKE_Q(wake_q); in futex_lock_pi()
1034 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q); in futex_lock_pi()
1035 raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q); in futex_lock_pi()
H A Dfutex.h163 typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
268 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
/linux/io_uring/
H A Dfutex.c152 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) in io_futex_wakev_fn() argument
203 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) in io_futex_wake_fn() argument
/linux/include/linux/
H A Dsched.h1232 struct wake_q_node wake_q; member
/linux/kernel/sched/
H A Dcore.c1006 struct wake_q_node *node = &task->wake_q; in __wake_q_add()
1076 task = container_of(node, struct task_struct, wake_q); in wake_up_q()
1079 WRITE_ONCE(task->wake_q.next, NULL); in wake_up_q()
/linux/kernel/
H A Dfork.c929 tsk->wake_q.next = NULL; in dup_task_struct()