kernel/sched/ext.c

20 static struct scx_sched __rcu *scx_root;
199 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
200 static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
203 static __printf(4, 5) bool scx_exit(struct scx_sched *sch,
248 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch,
254 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)
386 static __always_inline bool scx_kf_allowed(struct scx_sched *sch, u32 mask)
417 static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch,
683  * @sch: scx_sched to account events for
696  * @sch: scx_sched to account events for
728 static void scx_read_events(struct scx_sched *sch,
773  * @sch: scx_sched to abort on error
781 static bool ops_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)
793  * @sch: scx_sched to error out on error
804 static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err)
987 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
1017 static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
1214 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
1246 static void mark_direct_dispatch(struct scx_sched *sch,
1298 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
1366 	struct scx_sched *sch = scx_root;
1491 	struct scx_sched *sch = scx_root;
1541 	struct scx_sched *sch = scx_root;
1590 	struct scx_sched *sch = scx_root;
1635 	struct scx_sched *sch = scx_root;
1646 	struct scx_sched *sch = scx_root;
1732 static bool task_can_run_on_remote_rq(struct scx_sched *sch,
1847  * @sch: scx_sched being operated on
1861 static struct rq *move_task_between_dsqs(struct scx_sched *sch,
1914 static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
1961 static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq)
1970  * @sch: scx_sched being operated on
1983 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
2083 static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
2145 static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
2176 	struct scx_sched *sch = scx_root;
2311 		struct scx_sched *sch = scx_root;
2327 	struct scx_sched *sch = scx_root;
2383 	struct scx_sched *sch = scx_root;
2424 	struct scx_sched *sch = scx_root;
2578 			struct scx_sched *sch = rcu_dereference_sched(scx_root);
2646 	struct scx_sched *sch = scx_root;
2666 	struct scx_sched *sch = scx_root;
2727 	struct scx_sched *sch = scx_root;
2749 	struct scx_sched *sch = scx_root;
2799 	struct scx_sched *sch;
2846 	struct scx_sched *sch;
2871 	struct scx_sched *sch = scx_root;
2950 	struct scx_sched *sch = scx_root;
3003 	struct scx_sched *sch = scx_root;
3031 	struct scx_sched *sch = scx_root;
3046 	struct scx_sched *sch = scx_root;
3211 	struct scx_sched *sch = scx_root;
3230 	struct scx_sched *sch = scx_root;
3305 	struct scx_sched *sch = scx_root;
3334 	struct scx_sched *sch = scx_root;
3347 	struct scx_sched *sch = scx_root;
3396 	struct scx_sched *sch = scx_root;
3415 	struct scx_sched *sch = scx_root;
3433 	struct scx_sched *sch = scx_root;
3449 	struct scx_sched *sch = scx_root;
3466 	struct scx_sched *sch = scx_root;
3570 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)
3610 static void scx_cgroup_exit(struct scx_sched *sch)
3635 static int scx_cgroup_init(struct scx_sched *sch)
3678 static void scx_cgroup_exit(struct scx_sched *sch) {}
3679 static int scx_cgroup_init(struct scx_sched *sch) { return 0; }
3746 	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
3778 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
3787 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
3800 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
3823 ATTRIBUTE_GROUPS(scx_sched);
3833 	const struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
3857 	struct scx_sched *sch;
3888 	struct scx_sched *sch;
3968 static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
4064 static void bypass_lb_node(struct scx_sched *sch, int node)
4146 	struct scx_sched *sch;
4199 	struct scx_sched *sch;
4360 	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
4513 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
4533 	struct scx_sched *sch;
4663 	struct scx_sched *sch = scx_root;
4706 	struct scx_sched *sch = scx_root;
4852 	struct scx_sched *sch = container_of(irq_work, struct scx_sched, error_irq_work);
4861 static bool scx_vexit(struct scx_sched *sch,
4917 static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
4919 	struct scx_sched *sch;
4999 static int check_hotplug_seq(struct scx_sched *sch,
5023 static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
5071 	struct scx_sched *sch;
5491 	struct scx_sched *sch = ops->priv;
5775 	struct scx_sched *sch = scx_root;
5879 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p,
5900 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
5969 	struct scx_sched *sch;
5998 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
6058 	struct scx_sched *sch;
6076 	struct scx_sched *sch;
6104 	struct scx_sched *sch = scx_root;
6197 	struct scx_sched *sch;
6220 	struct scx_sched *sch;
6255 	struct scx_sched *sch;
6456 	struct scx_sched *sch;
6497 	struct scx_sched *sch;
6573 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
6638 	struct scx_sched *sch;
6655 	struct scx_sched *sch;
6701 	struct scx_sched *sch;
6724 	struct scx_sched *sch;
6838 	struct scx_sched *sch;
6861 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
6898 static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
6920 	struct scx_sched *sch;
6943 	struct scx_sched *sch;
6969 	struct scx_sched *sch;
7040 	struct scx_sched *sch;
7067 	struct scx_sched *sch;
7094 	struct scx_sched *sch;
7211 	struct scx_sched *sch;
7240 	struct scx_sched *sch;
7266 	struct scx_sched *sch;
7296 	struct scx_sched *sch;
7377 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events)
7406 	struct scx_sched *sch;