Lines Matching refs:reenq_flags

1150 			       u64 reenq_flags, struct rq *locked_rq)  in schedule_dsq_reenq()  argument
1174 (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { in schedule_dsq_reenq()
1180 WRITE_ONCE(drl->flags, drl->flags | reenq_flags); in schedule_dsq_reenq()
1195 (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { in schedule_dsq_reenq()
1201 WRITE_ONCE(dru->flags, dru->flags | reenq_flags); in schedule_dsq_reenq()
1214 static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) in schedule_reenq_local() argument
1221 schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); in schedule_reenq_local()
3908 static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) in local_task_should_reenq() argument
3912 first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); in local_task_should_reenq()
3913 *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; in local_task_should_reenq()
3918 (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { in local_task_should_reenq()
3924 return *reenq_flags & SCX_REENQ_ANY; in local_task_should_reenq()
3927 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) in reenq_local() argument
3935 if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) in reenq_local()
3936 reenq_flags &= ~__SCX_REENQ_TSR_MASK; in reenq_local()
3938 reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; in reenq_local()
3967 if (!local_task_should_reenq(p, &reenq_flags, &reason)) in reenq_local()
3999 u64 reenq_flags; in process_deferred_reenq_locals() local
4016 reenq_flags = drl->flags; in process_deferred_reenq_locals()
4038 reenq_local(sch, rq, reenq_flags); in process_deferred_reenq_locals()
4043 static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) in user_task_should_reenq() argument
4046 return reenq_flags & SCX_REENQ_ANY; in user_task_should_reenq()
4049 static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) in reenq_user() argument
4069 if (!user_task_should_reenq(p, reenq_flags, &reason)) in reenq_user()
4126 u64 reenq_flags; in process_deferred_reenq_users() local
4141 reenq_flags = dru->flags; in process_deferred_reenq_users()
4150 reenq_user(rq, dsq, reenq_flags); in process_deferred_reenq_users()
8835 __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, in scx_bpf_dsq_reenq() argument
8847 if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { in scx_bpf_dsq_reenq()
8848 scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); in scx_bpf_dsq_reenq()
8853 if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) in scx_bpf_dsq_reenq()
8854 reenq_flags |= SCX_REENQ_ANY; in scx_bpf_dsq_reenq()
8857 schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); in scx_bpf_dsq_reenq()