10  * scheduler will round-robin between clients to submit the next job.
13  * jobs when bulk background jobs are queued up, we submit a new job
71 	struct v3d_job *job = to_v3d_job(sched_job);  in v3d_sched_job_free()  local
73 	v3d_job_cleanup(job);  in v3d_sched_job_free()
79 	struct v3d_cpu_job *job = to_cpu_job(sched_job);  in v3d_cpu_job_free()  local
80 	struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;  in v3d_cpu_job_free()
81 	struct v3d_performance_query_info *performance_query = &job->performance_query;  in v3d_cpu_job_free()
95 	v3d_job_cleanup(&job->base);  in v3d_cpu_job_free()
99 v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)  in v3d_switch_perfmon()  argument
101 	if (job->perfmon != v3d->active_perfmon)  in v3d_switch_perfmon()
104 	if (job->perfmon && v3d->active_perfmon != job->perfmon)  in v3d_switch_perfmon()
105 		v3d_perfmon_start(v3d, job->perfmon);  in v3d_switch_perfmon()
110 	struct v3d_bin_job *job = to_bin_job(sched_job);  in v3d_bin_job_run()  local
111 	struct v3d_dev *v3d = job->base.v3d;  in v3d_bin_job_run()
112 	struct v3d_file_priv *file = job->base.file->driver_priv;  in v3d_bin_job_run()
117 	if (unlikely(job->base.base.s_fence->finished.error))  in v3d_bin_job_run()
124 	v3d->bin_job = job;  in v3d_bin_job_run()
126 	 * reuse the overflow attached to a previous job.  in v3d_bin_job_run()
137 	if (job->base.irq_fence)  in v3d_bin_job_run()
138 		dma_fence_put(job->base.irq_fence);  in v3d_bin_job_run()
139 	job->base.irq_fence = dma_fence_get(fence);  in v3d_bin_job_run()
142 			    job->start, job->end);  in v3d_bin_job_run()
147 	v3d_switch_perfmon(v3d, &job->base);  in v3d_bin_job_run()
150 	 * Writing the end register is what starts the job.  in v3d_bin_job_run()
152 	if (job->qma) {  in v3d_bin_job_run()
153 		V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);  in v3d_bin_job_run()
154 		V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);  in v3d_bin_job_run()
156 	if (job->qts) {  in v3d_bin_job_run()
159 			       job->qts);  in v3d_bin_job_run()
161 	V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);  in v3d_bin_job_run()
162 	V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);  in v3d_bin_job_run()
169 	struct v3d_render_job *job = to_render_job(sched_job);  in v3d_render_job_run()  local
170 	struct v3d_dev *v3d = job->base.v3d;  in v3d_render_job_run()
171 	struct v3d_file_priv *file = job->base.file->driver_priv;  in v3d_render_job_run()
175 	if (unlikely(job->base.base.s_fence->finished.error))  in v3d_render_job_run()
178 	v3d->render_job = job;  in v3d_render_job_run()
192 	if (job->base.irq_fence)  in v3d_render_job_run()
193 		dma_fence_put(job->base.irq_fence);  in v3d_render_job_run()
194 	job->base.irq_fence = dma_fence_get(fence);  in v3d_render_job_run()
197 			    job->start, job->end);  in v3d_render_job_run()
202 	v3d_switch_perfmon(v3d, &job->base);  in v3d_render_job_run()
207 	 * Writing the end register is what starts the job.  in v3d_render_job_run()
209 	V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);  in v3d_render_job_run()
210 	V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);  in v3d_render_job_run()
218 	struct v3d_tfu_job *job = to_tfu_job(sched_job);  in v3d_tfu_job_run()  local
219 	struct v3d_dev *v3d = job->base.v3d;  in v3d_tfu_job_run()
220 	struct v3d_file_priv *file = job->base.file->driver_priv;  in v3d_tfu_job_run()
228 	v3d->tfu_job = job;  in v3d_tfu_job_run()
229 	if (job->base.irq_fence)  in v3d_tfu_job_run()
230 		dma_fence_put(job->base.irq_fence);  in v3d_tfu_job_run()
231 	job->base.irq_fence = dma_fence_get(fence);  in v3d_tfu_job_run()
238 	V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);  in v3d_tfu_job_run()
239 	V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);  in v3d_tfu_job_run()
240 	V3D_WRITE(V3D_TFU_ICA(v3d->ver), job->args.ica);  in v3d_tfu_job_run()
241 	V3D_WRITE(V3D_TFU_IUA(v3d->ver), job->args.iua);  in v3d_tfu_job_run()
242 	V3D_WRITE(V3D_TFU_IOA(v3d->ver), job->args.ioa);  in v3d_tfu_job_run()
244 		V3D_WRITE(V3D_V7_TFU_IOC, job->args.v71.ioc);  in v3d_tfu_job_run()
245 	V3D_WRITE(V3D_TFU_IOS(v3d->ver), job->args.ios);  in v3d_tfu_job_run()
246 	V3D_WRITE(V3D_TFU_COEF0(v3d->ver), job->args.coef[0]);  in v3d_tfu_job_run()
247 	if (v3d->ver >= 71 || (job->args.coef[0] & V3D_TFU_COEF0_USECOEF)) {  in v3d_tfu_job_run()
248 		V3D_WRITE(V3D_TFU_COEF1(v3d->ver), job->args.coef[1]);  in v3d_tfu_job_run()
249 		V3D_WRITE(V3D_TFU_COEF2(v3d->ver), job->args.coef[2]);  in v3d_tfu_job_run()
250 		V3D_WRITE(V3D_TFU_COEF3(v3d->ver), job->args.coef[3]);  in v3d_tfu_job_run()
252 	/* ICFG kicks off the job. */  in v3d_tfu_job_run()
253 	V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);  in v3d_tfu_job_run()
261 	struct v3d_csd_job *job = to_csd_job(sched_job);  in v3d_csd_job_run()  local
262 	struct v3d_dev *v3d = job->base.v3d;  in v3d_csd_job_run()
263 	struct v3d_file_priv *file = job->base.file->driver_priv;  in v3d_csd_job_run()
268 	v3d->csd_job = job;  in v3d_csd_job_run()
276 	if (job->base.irq_fence)  in v3d_csd_job_run()
277 		dma_fence_put(job->base.irq_fence);  in v3d_csd_job_run()
278 	job->base.irq_fence = dma_fence_get(fence);  in v3d_csd_job_run()
285 	v3d_switch_perfmon(v3d, &job->base);  in v3d_csd_job_run()
290 		V3D_CORE_WRITE(0, csd_cfg0_reg + 4 * i, job->args.cfg[i]);  in v3d_csd_job_run()
291 	/* CFG0 write kicks off the job. */  in v3d_csd_job_run()
292 	V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);  in v3d_csd_job_run()
298 v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job)  in v3d_rewrite_csd_job_wg_counts_from_indirect()  argument
300 	struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;  in v3d_rewrite_csd_job_wg_counts_from_indirect()
301 	struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);  in v3d_rewrite_csd_job_wg_counts_from_indirect()
303 	struct drm_v3d_submit_csd *args = &indirect_csd->job->args;  in v3d_rewrite_csd_job_wg_counts_from_indirect()
333 v3d_timestamp_query(struct v3d_cpu_job *job)  in v3d_timestamp_query()  argument
335 	struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;  in v3d_timestamp_query()
336 	struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);  in v3d_timestamp_query()
346 					  job->base.done_fence);  in v3d_timestamp_query()
353 v3d_reset_timestamp_queries(struct v3d_cpu_job *job)  in v3d_reset_timestamp_queries()  argument
355 	struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;  in v3d_reset_timestamp_queries()
357 	struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);  in v3d_reset_timestamp_queries()
387 v3d_copy_query_results(struct v3d_cpu_job *job)  in v3d_copy_query_results()  argument
389 	struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;  in v3d_copy_query_results()
391 	struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);  in v3d_copy_query_results()
392 	struct v3d_bo *timestamp = to_v3d_bo(job->base.bo[1]);  in v3d_copy_query_results()
393 	struct v3d_copy_query_results_info *copy = &job->copy;  in v3d_copy_query_results()
428 v3d_reset_performance_queries(struct v3d_cpu_job *job)  in v3d_reset_performance_queries()  argument
430 	struct v3d_performance_query_info *performance_query = &job->performance_query;  in v3d_reset_performance_queries()
431 	struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;  in v3d_reset_performance_queries()
432 	struct v3d_dev *v3d = job->base.v3d;  in v3d_reset_performance_queries()
456 v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data, u32 query)  in v3d_write_performance_query_result()  argument
458 	struct v3d_performance_query_info *performance_query = &job->performance_query;  in v3d_write_performance_query_result()
459 	struct v3d_copy_query_results_info *copy = &job->copy;  in v3d_write_performance_query_result()
460 	struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;  in v3d_write_performance_query_result()
461 	struct v3d_dev *v3d = job->base.v3d;  in v3d_write_performance_query_result()
486 v3d_copy_performance_query(struct v3d_cpu_job *job)  in v3d_copy_performance_query()  argument
488 	struct v3d_performance_query_info *performance_query = &job->performance_query;  in v3d_copy_performance_query()
489 	struct v3d_copy_query_results_info *copy = &job->copy;  in v3d_copy_performance_query()
490 	struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);  in v3d_copy_performance_query()
505 			v3d_write_performance_query_result(job, data, i);  in v3d_copy_performance_query()
531 	struct v3d_cpu_job *job = to_cpu_job(sched_job);  in v3d_cpu_job_run()  local
532 	struct v3d_dev *v3d = job->base.v3d;  in v3d_cpu_job_run()
533 	struct v3d_file_priv *file = job->base.file->driver_priv;  in v3d_cpu_job_run()
536 	v3d->cpu_job = job;  in v3d_cpu_job_run()
538 	if (job->job_type >= ARRAY_SIZE(cpu_job_function)) {  in v3d_cpu_job_run()
539 		DRM_DEBUG_DRIVER("Unknown CPU job: %d\n", job->job_type);  in v3d_cpu_job_run()
546 	trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);  in v3d_cpu_job_run()
548 	cpu_job_function[job->job_type](job);  in v3d_cpu_job_run()
550 	trace_v3d_cpu_job_end(&v3d->drm, job->job_type);  in v3d_cpu_job_run()
569 	struct v3d_job *job = to_v3d_job(sched_job);  in v3d_cache_clean_job_run()  local
570 	struct v3d_dev *v3d = job->v3d;  in v3d_cache_clean_job_run()
571 	struct v3d_file_priv *file = job->file->driver_priv;  in v3d_cache_clean_job_run()
632 	struct v3d_job *job = to_v3d_job(sched_job);  in v3d_cl_job_timedout()  local
633 	struct v3d_dev *v3d = job->v3d;  in v3d_cl_job_timedout()
649 	struct v3d_bin_job *job = to_bin_job(sched_job);  in v3d_bin_job_timedout()  local
652 				   &job->timedout_ctca, &job->timedout_ctra);  in v3d_bin_job_timedout()
658 	struct v3d_render_job *job = to_render_job(sched_job);  in v3d_render_job_timedout()  local
661 				   &job->timedout_ctca, &job->timedout_ctra);  in v3d_render_job_timedout()
667 	struct v3d_job *job = to_v3d_job(sched_job);  in v3d_generic_job_timedout()  local
669 	return v3d_gpu_reset_for_timeout(job->v3d, sched_job);  in v3d_generic_job_timedout()
675 	struct v3d_csd_job *job = to_csd_job(sched_job);  in v3d_csd_job_timedout()  local
676 	struct v3d_dev *v3d = job->base.v3d;  in v3d_csd_job_timedout()
682 	if (job->timedout_batches != batches) {  in v3d_csd_job_timedout()
683 		job->timedout_batches = batches;  in v3d_csd_job_timedout()