44  * The jobs in a entity are always scheduled in the order that they were pushed.
47  * hardware, i.e. the pending queue, the entity must not be referenced anymore
48  * through the jobs entity pointer.
116  * @entity: the scheduler entity
118  * Return true if we can push at least one more job from @entity, false
122 				struct drm_sched_entity *entity)  in drm_sched_can_queue()  argument
126 	s_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));  in drm_sched_can_queue()
156 static inline void drm_sched_rq_remove_fifo_locked(struct drm_sched_entity *entity)  in drm_sched_rq_remove_fifo_locked()  argument
158 	struct drm_sched_rq *rq = entity->rq;  in drm_sched_rq_remove_fifo_locked()
160 	if (!RB_EMPTY_NODE(&entity->rb_tree_node)) {  in drm_sched_rq_remove_fifo_locked()
161 		rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root);  in drm_sched_rq_remove_fifo_locked()
162 		RB_CLEAR_NODE(&entity->rb_tree_node);  in drm_sched_rq_remove_fifo_locked()
166 void drm_sched_rq_update_fifo(struct drm_sched_entity *entity, ktime_t ts)  in drm_sched_rq_update_fifo()  argument
169 	 * Both locks need to be grabbed, one to protect from entity->rq change  in drm_sched_rq_update_fifo()
170 	 * for entity from within concurrent drm_sched_entity_select_rq and the  in drm_sched_rq_update_fifo()
173 	spin_lock(&entity->rq_lock);  in drm_sched_rq_update_fifo()
174 	spin_lock(&entity->rq->lock);  in drm_sched_rq_update_fifo()
176 	drm_sched_rq_remove_fifo_locked(entity);  in drm_sched_rq_update_fifo()
178 	entity->oldest_job_waiting = ts;  in drm_sched_rq_update_fifo()
180 	rb_add_cached(&entity->rb_tree_node, &entity->rq->rb_tree_root,  in drm_sched_rq_update_fifo()
183 	spin_unlock(&entity->rq->lock);  in drm_sched_rq_update_fifo()
184 	spin_unlock(&entity->rq_lock);  in drm_sched_rq_update_fifo()
206  * drm_sched_rq_add_entity - add an entity
209  * @entity: scheduler entity
211  * Adds a scheduler entity to the run queue.
214 			     struct drm_sched_entity *entity)  in drm_sched_rq_add_entity()  argument
216 	if (!list_empty(&entity->list))  in drm_sched_rq_add_entity()
222 	list_add_tail(&entity->list, &rq->entities);  in drm_sched_rq_add_entity()
228  * drm_sched_rq_remove_entity - remove an entity
231  * @entity: scheduler entity
233  * Removes a scheduler entity from the run queue.
236 				struct drm_sched_entity *entity)  in drm_sched_rq_remove_entity()  argument
238 	if (list_empty(&entity->list))  in drm_sched_rq_remove_entity()
244 	list_del_init(&entity->list);  in drm_sched_rq_remove_entity()
246 	if (rq->current_entity == entity)  in drm_sched_rq_remove_entity()
250 		drm_sched_rq_remove_fifo_locked(entity);  in drm_sched_rq_remove_entity()
256  * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run
261  * Try to find the next ready entity.
263  * Return an entity if one is found; return an error-pointer (!NULL) if an
264  * entity was ready, but the scheduler had insufficient credits to accommodate
265  * its job; return NULL, if no ready entity was found.
271 	struct drm_sched_entity *entity;  in drm_sched_rq_select_entity_rr()  local
275 	entity = rq->current_entity;  in drm_sched_rq_select_entity_rr()
276 	if (entity) {  in drm_sched_rq_select_entity_rr()
277 		list_for_each_entry_continue(entity, &rq->entities, list) {  in drm_sched_rq_select_entity_rr()
278 			if (drm_sched_entity_is_ready(entity)) {  in drm_sched_rq_select_entity_rr()
280 				 * entity in terms of fairness.  in drm_sched_rq_select_entity_rr()
282 				if (!drm_sched_can_queue(sched, entity)) {  in drm_sched_rq_select_entity_rr()
287 				rq->current_entity = entity;  in drm_sched_rq_select_entity_rr()
288 				reinit_completion(&entity->entity_idle);  in drm_sched_rq_select_entity_rr()
290 				return entity;  in drm_sched_rq_select_entity_rr()
295 	list_for_each_entry(entity, &rq->entities, list) {  in drm_sched_rq_select_entity_rr()
296 		if (drm_sched_entity_is_ready(entity)) {  in drm_sched_rq_select_entity_rr()
297 			/* If we can't queue yet, preserve the current entity in  in drm_sched_rq_select_entity_rr()
300 			if (!drm_sched_can_queue(sched, entity)) {  in drm_sched_rq_select_entity_rr()
305 			rq->current_entity = entity;  in drm_sched_rq_select_entity_rr()
306 			reinit_completion(&entity->entity_idle);  in drm_sched_rq_select_entity_rr()
308 			return entity;  in drm_sched_rq_select_entity_rr()
311 		if (entity == rq->current_entity)  in drm_sched_rq_select_entity_rr()
321  * drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run
326  * Find oldest waiting ready entity.
328  * Return an entity if one is found; return an error-pointer (!NULL) if an
329  * entity was ready, but the scheduler had insufficient credits to accommodate
330  * its job; return NULL, if no ready entity was found.
340 		struct drm_sched_entity *entity;  in drm_sched_rq_select_entity_fifo()  local
342 		entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node);  in drm_sched_rq_select_entity_fifo()
343 		if (drm_sched_entity_is_ready(entity)) {  in drm_sched_rq_select_entity_fifo()
344 			/* If we can't queue yet, preserve the current entity in  in drm_sched_rq_select_entity_fifo()
347 			if (!drm_sched_can_queue(sched, entity)) {  in drm_sched_rq_select_entity_fifo()
352 			rq->current_entity = entity;  in drm_sched_rq_select_entity_fifo()
353 			reinit_completion(&entity->entity_idle);  in drm_sched_rq_select_entity_fifo()
773  * @entity: scheduler entity to use
785  * has died, which can mean that there's no valid runqueue for a @entity.
792 		       struct drm_sched_entity *entity,  in drm_sched_job_init()  argument
795 	if (!entity->rq) {  in drm_sched_job_init()
800 		drm_err(job->sched, "%s: entity has no rq!\n", __func__);  in drm_sched_job_init()
809 	job->entity = entity;  in drm_sched_job_init()
811 	job->s_fence = drm_sched_fence_alloc(entity, owner);  in drm_sched_job_init()
839 	struct drm_sched_entity *entity = job->entity;  in drm_sched_job_arm()  local
841 	BUG_ON(!entity);  in drm_sched_job_arm()
842 	drm_sched_entity_select_rq(entity);  in drm_sched_job_arm()
843 	sched = entity->rq->sched;  in drm_sched_job_arm()
846 	job->s_priority = entity->priority;  in drm_sched_job_arm()
849 	drm_sched_fence_init(job->s_fence, job->entity);  in drm_sched_job_arm()
1025  * @entity: the scheduler entity
1030 		      struct drm_sched_entity *entity)  in drm_sched_wakeup()  argument
1032 	if (drm_sched_can_queue(sched, entity))  in drm_sched_wakeup()
1037  * drm_sched_select_entity - Select next entity to process
1041  * Return an entity to process or NULL if none are found.
1043  * Note, that we break out of the for-loop when "entity" is non-null, which can
1050 	struct drm_sched_entity *entity;  in drm_sched_select_entity()  local
1056 		entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ?  in drm_sched_select_entity()
1059 		if (entity)  in drm_sched_select_entity()
1063 	return IS_ERR(entity) ? NULL : entity;  in drm_sched_select_entity()
1178 	struct drm_sched_entity *entity;  in drm_sched_run_job_work()  local
1187 	/* Find entity with a ready job */  in drm_sched_run_job_work()
1188 	entity = drm_sched_select_entity(sched);  in drm_sched_run_job_work()
1189 	if (!entity)  in drm_sched_run_job_work()
1192 	sched_job = drm_sched_entity_pop_job(entity);  in drm_sched_run_job_work()
1194 		complete_all(&entity->entity_idle);  in drm_sched_run_job_work()
1204 	trace_drm_run_job(sched_job, entity);  in drm_sched_run_job_work()
1206 	complete_all(&entity->entity_idle);  in drm_sched_run_job_work()
1378  * limit of the scheduler then the respective sched entity is marked guilty and
1385 	struct drm_sched_entity *entity;  in drm_sched_increase_karma()  local
1399 			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {  in drm_sched_increase_karma()
1401 				    entity->fence_context) {  in drm_sched_increase_karma()
1402 					if (entity->guilty)  in drm_sched_increase_karma()
1403 						atomic_set(entity->guilty, 1);  in drm_sched_increase_karma()
1408 			if (&entity->list != &rq->entities)  in drm_sched_increase_karma()