106 int panfrost_job_get_slot(struct panfrost_job *job)  in panfrost_job_get_slot()  argument
112 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
117 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
118 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
119 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
121 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
146 panfrost_get_job_chain_flag(const struct panfrost_job *job)  in panfrost_get_job_chain_flag()  argument
148 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);  in panfrost_get_job_chain_flag()
150 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
159 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()  local
161 	WARN_ON(!job);  in panfrost_dequeue_job()
162 	if (job->is_profiled) {  in panfrost_dequeue_job()
163 		if (job->engine_usage) {  in panfrost_dequeue_job()
164 			job->engine_usage->elapsed_ns[slot] +=  in panfrost_dequeue_job()
165 				ktime_to_ns(ktime_sub(ktime_get(), job->start_time));  in panfrost_dequeue_job()
166 			job->engine_usage->cycles[slot] +=  in panfrost_dequeue_job()
167 				panfrost_cycle_counter_read(pfdev) - job->start_cycles;  in panfrost_dequeue_job()
169 		panfrost_cycle_counter_put(job->pfdev);  in panfrost_dequeue_job()
175 	return job;  in panfrost_dequeue_job()
180 		     struct panfrost_job *job)  in panfrost_enqueue_job()  argument
182 	if (WARN_ON(!job))  in panfrost_enqueue_job()
186 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
191 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
192 	WARN_ON(panfrost_get_job_chain_flag(job) ==  in panfrost_enqueue_job()
197 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)  in panfrost_job_hw_submit()  argument
199 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
202 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
215 	cfg = panfrost_mmu_as_get(pfdev, job->mmu);  in panfrost_job_hw_submit()
220 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
227 		panfrost_get_job_chain_flag(job);  in panfrost_job_hw_submit()
238 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
243 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
244 	/* Don't queue the job if a reset is in progress */  in panfrost_job_hw_submit()
248 			job->is_profiled = true;  in panfrost_job_hw_submit()
249 			job->start_time = ktime_get();  in panfrost_job_hw_submit()
250 			job->start_cycles = panfrost_cycle_counter_read(pfdev);  in panfrost_job_hw_submit()
256 			job, js, subslot, jc_head, cfg & 0xf);  in panfrost_job_hw_submit()
263 					  struct drm_sched_job *job)  in panfrost_acquire_object_fences()  argument
273 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],  in panfrost_acquire_object_fences()
292 int panfrost_job_push(struct panfrost_job *job)  in panfrost_job_push()  argument
294 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
298 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
304 	drm_sched_job_arm(&job->base);  in panfrost_job_push()
306 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
308 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
309 					     &job->base);  in panfrost_job_push()
315 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
317 	drm_sched_entity_push_job(&job->base);  in panfrost_job_push()
321 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
322 				      job->render_done_fence);  in panfrost_job_push()
325 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
332 	struct panfrost_job *job = container_of(ref, struct panfrost_job,  in panfrost_job_cleanup()  local
336 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
337 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
339 	if (job->mappings) {  in panfrost_job_cleanup()
340 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
341 			if (!job->mappings[i])  in panfrost_job_cleanup()
344 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
345 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
347 		kvfree(job->mappings);  in panfrost_job_cleanup()
350 	if (job->bos) {  in panfrost_job_cleanup()
351 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
352 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
354 		kvfree(job->bos);  in panfrost_job_cleanup()
357 	kfree(job);  in panfrost_job_cleanup()
360 void panfrost_job_put(struct panfrost_job *job)  in panfrost_job_put()  argument
362 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
367 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_free()  local
371 	panfrost_job_put(job);  in panfrost_job_free()
376 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_run()  local
377 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
378 	int slot = panfrost_job_get_slot(job);  in panfrost_job_run()
381 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
384 	/* Nothing to execute: can happen if the job has finished while  in panfrost_job_run()
387 	if (!job->jc)  in panfrost_job_run()
394 	if (job->done_fence)  in panfrost_job_run()
395 		dma_fence_put(job->done_fence);  in panfrost_job_run()
396 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
398 	panfrost_job_hw_submit(job, slot);  in panfrost_job_run()
427 				    struct panfrost_job *job,  in panfrost_job_handle_err()  argument
447 		/* Update the job head so we can resume */  in panfrost_job_handle_err()
448 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
451 		/* The job will be resumed, don't signal the fence */  in panfrost_job_handle_err()
454 		/* Job has been hard-stopped, flag it as canceled */  in panfrost_job_handle_err()
455 		dma_fence_set_error(job->done_fence, -ECANCELED);  in panfrost_job_handle_err()
456 		job->jc = 0;  in panfrost_job_handle_err()
462 		dma_fence_set_error(job->done_fence, -EINVAL);  in panfrost_job_handle_err()
463 		job->jc = 0;  in panfrost_job_handle_err()
466 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_err()
470 		dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_err()
481 				     struct panfrost_job *job)  in panfrost_job_handle_done()  argument
483 	/* Set ->jc to 0 to avoid re-submitting an already finished job (can  in panfrost_job_handle_done()
486 	job->jc = 0;  in panfrost_job_handle_done()
487 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_done()
490 	dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_done()
530 		 * is racy. If we only have one job done at the time we  in panfrost_job_handle_irq()
531 		 * read JOB_INT_RAWSTAT but the second job fails before we  in panfrost_job_handle_irq()
553 			/* When the current job doesn't fail, the JM dequeues  in panfrost_job_handle_irq()
554 			 * the next job without waiting for an ACK, this means  in panfrost_job_handle_irq()
557 			 * are inactive, but one job remains in pfdev->jobs[j],  in panfrost_job_handle_irq()
560 			 * job in _NEXT (see above).  in panfrost_job_handle_irq()
583 			/* The job was cancelled, signal the fence now */  in panfrost_job_handle_irq()
589 			/* Requeue the job we removed if no reset is pending */  in panfrost_job_handle_irq()
646 	 * to release job resources. We should rework the code to follow this  in panfrost_reset()
663 	/* Mask job interrupts and synchronize to make sure we won't be  in panfrost_reset()
670 		/* Cancel the next job and soft-stop the running job. */  in panfrost_reset()
689 	 * panfrost_devfreq_record_idle() for each stuck job.  in panfrost_reset()
708 	/* panfrost_device_reset() unmasks job interrupts, but we want to  in panfrost_reset()
719 	 * while resubmitting jobs because the job submission logic will  in panfrost_reset()
732 	/* Re-enable job interrupts now that everything has been restarted. */  in panfrost_reset()
743 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_timedout()  local
744 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
745 	int js = panfrost_job_get_slot(job);  in panfrost_job_timedout()
751 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
760 	 * job timeouts, synchronize the IRQ handler and re-check the fence  in panfrost_job_timedout()
765 	if (dma_fence_is_signaled(job->done_fence)) {  in panfrost_job_timedout()
778 	panfrost_core_dump(job);  in panfrost_job_timedout()
838 	 * disambiguation stopping the right job in the close path is tricky,  in panfrost_job_init()
851 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
858 					IRQF_SHARED, KBUILD_MODNAME "-job",  in panfrost_job_init()
861 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
945 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()  local
948 			if (!job || job->base.entity != entity)  in panfrost_job_close()
952 				/* Try to cancel the job before it starts */  in panfrost_job_close()
954 				/* Reset the job head so it doesn't get restarted if  in panfrost_job_close()
955 				 * the job in the first slot failed.  in panfrost_job_close()
957 				job->jc = 0;  in panfrost_job_close()
961 				cmd = panfrost_get_job_chain_flag(job) ?  in panfrost_job_close()
971 			job->engine_usage = NULL;  in panfrost_job_close()