kernel/sched/fair.c

290 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
410 is_same_group(struct sched_entity *se, struct sched_entity *pse)
418 static inline struct sched_entity *parent_entity(const struct sched_entity *se)
424 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
465 static int se_is_idle(struct sched_entity *se)
493 static inline struct sched_entity *parent_entity(struct sched_entity *se)
499 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
513 static int se_is_idle(struct sched_entity *se)
545 static inline bool entity_before(const struct sched_entity *a,
546 				 const struct sched_entity *b)
555 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
561 	rb_entry((node), struct sched_entity, run_node)
622 avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
632 avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
656 	struct sched_entity *curr = cfs_rq->curr;
693 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
724 	struct sched_entity *curr = cfs_rq->curr;
738 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
759 	struct sched_entity *se = __pick_root_entity(cfs_rq);
760 	struct sched_entity *curr = cfs_rq->curr;
783 	struct sched_entity *root = __pick_root_entity(cfs_rq);
784 	struct sched_entity *curr = cfs_rq->curr;
803 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
806 		struct sched_entity *rse = __node_2_se(node);
812 static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node)
815 		struct sched_entity *rse = __node_2_se(node);
824 static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
842 RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
848 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
857 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
864 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq)
874 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
892 static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
907 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
914 static inline bool protect_slice(struct sched_entity *se)
919 static inline void cancel_protect_slice(struct sched_entity *se)
944 static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect)
947 	struct sched_entity *se = __pick_first_entity(cfs_rq);
948 	struct sched_entity *curr = cfs_rq->curr;
949 	struct sched_entity *best = NULL;
1005 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
1010 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
1035 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
1041 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
1071 /* Give new sched_entity start runnable values to heavy its load in infant time */
1072 void init_entity_runnable_average(struct sched_entity *se)
1119 	struct sched_entity *se = &p->se;
1155 static s64 update_se(struct rq *rq, struct sched_entity *se)
1215 	struct sched_entity *curr = cfs_rq->curr;
1263 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1280 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1306 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1326 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1343 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1375 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
3680 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3693 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3752 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3759 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3768 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags);
3770 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
3826 	struct sched_entity *se = &p->se;
3932 	 * assigned as a minimum load.weight to the sched_entity representing
3948 static void update_cfs_group(struct sched_entity *se)
3969 static inline void update_cfs_group(struct sched_entity *se)
4161 void set_task_rq_fair(struct sched_entity *se,
4188  * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
4255 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4271 	/* Set new sched_entity's utilization */
4287 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4302 	/* Set new sched_entity's runnable */
4317 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4391 static inline int propagate_entity_load_avg(struct sched_entity *se)
4422 static inline bool skip_blocked_update(struct sched_entity *se)
4427 	 * If sched_entity still have not zero load or utilization, we have to
4435 	 * the utilization of the sched_entity:
4441 	 * Otherwise, the load and the utilization of the sched_entity is
4454 static inline int propagate_entity_load_avg(struct sched_entity *se)
4464 static inline void migrate_se_pelt_lag(struct sched_entity *se)
4543 static void migrate_se_pelt_lag(struct sched_entity *se) {}
4628  * @se: sched_entity to attach
4633 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4683  * @se: sched_entity to detach
4688 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4719 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4726 	 * track group sched_entity load average for task_h_load calculation in migration
4765 static void sync_entity_load_avg(struct sched_entity *se)
4778 static void remove_entity_load_avg(struct sched_entity *se)
5113 	struct sched_entity *se = &p->se;
5128 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5146 		struct sched_entity *curr = cfs_rq->curr;
5239 requeue_delayed_entity(struct sched_entity *se);
5242 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5256 	 * When enqueuing a sched_entity, we must:
5309 static void __clear_buddies_next(struct sched_entity *se)
5320 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
5328 static void set_delayed(struct sched_entity *se)
5349 static void clear_delayed(struct sched_entity *se)
5371 static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
5379 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5412 	 * When dequeuing a sched_entity, we must:
5460 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
5500 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags);
5509 static struct sched_entity *
5512 	struct sched_entity *se;
5537 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
5561 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
5803 	struct sched_entity *se;
5904 	struct sched_entity *se;
6690 	struct sched_entity *se = &p->se;
6788 requeue_delayed_entity(struct sched_entity *se)
6827 	struct sched_entity *se = &p->se;
6950 static void set_next_buddy(struct sched_entity *se);
6961 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
8550 	struct sched_entity *se = &p->se;
8576 	struct sched_entity *se = &p->se;
8632 static void set_next_buddy(struct sched_entity *se)
8649 	struct sched_entity *se = &donor->se, *pse = &p->se;
8742 	struct sched_entity *se;
8773 	struct sched_entity *se;
8801 		struct sched_entity *pse = &prev->se;
8886 	struct sched_entity *se = &prev->se;
8902 	struct sched_entity *se = &curr->se;
8929 	struct sched_entity *se = &p->se;
9703 		struct sched_entity *se;
9743 	struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
12888 __entity_slice_used(struct sched_entity *se, int min_nr_tasks)
12924 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq,
12942 	struct sched_entity *se = &p->se;
12954 	const struct sched_entity *sea = &a->se;
12955 	const struct sched_entity *seb = &b->se;
13024 	struct sched_entity *se = &curr->se;
13077  * Propagate the changes of the sched_entity across the tg tree to make it
13080 static void propagate_entity_cfs_rq(struct sched_entity *se)
13106 static void propagate_entity_cfs_rq(struct sched_entity *se) { }
13109 static void detach_entity_cfs_rq(struct sched_entity *se)
13129 static void attach_entity_cfs_rq(struct sched_entity *se)
13142 	struct sched_entity *se = &p->se;
13149 	struct sched_entity *se = &p->se;
13182 	struct sched_entity *se = &p->se;
13211 	struct sched_entity *se = &p->se;
13266 	struct sched_entity *se;
13307 	struct sched_entity *se;
13331 		struct sched_entity *se = tg->se[cpu];
13358 			struct sched_entity *se, int cpu,
13359 			struct sched_entity *parent)
13410 		struct sched_entity *se = tg->se[i];
13461 		struct sched_entity *se = tg->se[i];
13510 	struct sched_entity *se = &task->se;