21 v3d_lock_bo_reservations(struct v3d_job *job,  in v3d_lock_bo_reservations()  argument
26 	ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_lock_bo_reservations()
30 	for (i = 0; i < job->bo_count; i++) {  in v3d_lock_bo_reservations()
31 		ret = dma_resv_reserve_fences(job->bo[i]->resv, 1);  in v3d_lock_bo_reservations()
35 		ret = drm_sched_job_add_implicit_dependencies(&job->base,  in v3d_lock_bo_reservations()
36 							      job->bo[i], true);  in v3d_lock_bo_reservations()
44 	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_lock_bo_reservations()
49  * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
50  * referenced by the job.
53  * @job: V3D job being set up
58  * the submitted job's BO list.  This does the validation of the job's
59  * BO list and reference counting for the lifetime of the job.
67 	       struct v3d_job *job,  in v3d_lookup_bos()  argument
71 	job->bo_count = bo_count;  in v3d_lookup_bos()
73 	if (!job->bo_count) {  in v3d_lookup_bos()
83 				      job->bo_count, &job->bo);  in v3d_lookup_bos()
89 	struct v3d_job *job = container_of(ref, struct v3d_job, refcount);  in v3d_job_free()  local
92 	if (job->bo) {  in v3d_job_free()
93 		for (i = 0; i < job->bo_count; i++)  in v3d_job_free()
94 			drm_gem_object_put(job->bo[i]);  in v3d_job_free()
95 		kvfree(job->bo);  in v3d_job_free()
98 	dma_fence_put(job->irq_fence);  in v3d_job_free()
99 	dma_fence_put(job->done_fence);  in v3d_job_free()
101 	if (job->perfmon)  in v3d_job_free()
102 		v3d_perfmon_put(job->perfmon);  in v3d_job_free()
104 	kfree(job);  in v3d_job_free()
110 	struct v3d_render_job *job = container_of(ref, struct v3d_render_job,  in v3d_render_job_free()  local
114 	list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {  in v3d_render_job_free()
121 void v3d_job_cleanup(struct v3d_job *job)  in v3d_job_cleanup()  argument
123 	if (!job)  in v3d_job_cleanup()
126 	drm_sched_job_cleanup(&job->base);  in v3d_job_cleanup()
127 	v3d_job_put(job);  in v3d_job_cleanup()
130 void v3d_job_put(struct v3d_job *job)  in v3d_job_put()  argument
132 	if (!job)  in v3d_job_put()
135 	kref_put(&job->refcount, job->free);  in v3d_job_put()
143 		DRM_ERROR("Cannot allocate memory for V3D job.\n");  in v3d_job_allocate()
159 	     struct v3d_job *job, void (*free)(struct kref *ref),  in v3d_job_init()  argument
166 	job->v3d = v3d;  in v3d_job_init()
167 	job->free = free;  in v3d_job_init()
168 	job->file = file_priv;  in v3d_job_init()
170 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],  in v3d_job_init()
187 				ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);  in v3d_job_init()
195 		ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0);  in v3d_job_init()
202 	kref_init(&job->refcount);  in v3d_job_init()
207 	drm_sched_job_cleanup(&job->base);  in v3d_job_init()
212 v3d_push_job(struct v3d_job *job)  in v3d_push_job()  argument
214 	drm_sched_job_arm(&job->base);  in v3d_push_job()
216 	job->done_fence = dma_fence_get(&job->base.s_fence->finished);  in v3d_push_job()
218 	/* put by scheduler job completion */  in v3d_push_job()
219 	kref_get(&job->refcount);  in v3d_push_job()
221 	drm_sched_entity_push_job(&job->base);  in v3d_push_job()
226 					 struct v3d_job *job,  in v3d_attach_fences_and_unlock_reservation()  argument
236 	for (i = 0; i < job->bo_count; i++) {  in v3d_attach_fences_and_unlock_reservation()
238 		dma_resv_add_fence(job->bo[i]->resv, job->done_fence,  in v3d_attach_fences_and_unlock_reservation()
242 	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_attach_fences_and_unlock_reservation()
244 	/* Update the return sync object for the job */  in v3d_attach_fences_and_unlock_reservation()
270 			   struct v3d_csd_job **job,  in v3d_setup_csd_jobs_and_bos()  argument
277 	ret = v3d_job_allocate((void *)job, sizeof(**job));  in v3d_setup_csd_jobs_and_bos()
281 	ret = v3d_job_init(v3d, file_priv, &(*job)->base,  in v3d_setup_csd_jobs_and_bos()
284 		v3d_job_deallocate((void *)job);  in v3d_setup_csd_jobs_and_bos()
299 	(*job)->args = *args;  in v3d_setup_csd_jobs_and_bos()
371  * to be signaled when job completes (out_sync).
382 		DRM_DEBUG("Two multisync extensions were added to the same job.");  in v3d_get_multisync_submit_deps()
405 /* Get data for the indirect CSD job submission. */
409 				struct v3d_cpu_job *job)  in v3d_get_cpu_indirect_csd_params()  argument
414 	struct v3d_indirect_csd_info *info = &job->indirect_csd;  in v3d_get_cpu_indirect_csd_params()
416 	if (!job) {  in v3d_get_cpu_indirect_csd_params()
417 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_indirect_csd_params()
421 	if (job->job_type) {  in v3d_get_cpu_indirect_csd_params()
422 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_indirect_csd_params()
434 	job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;  in v3d_get_cpu_indirect_csd_params()
443 					  &info->job, &info->clean_job,  in v3d_get_cpu_indirect_csd_params()
447 /* Get data for the query timestamp job submission. */
451 				   struct v3d_cpu_job *job)  in v3d_get_cpu_timestamp_query_params()  argument
456 	if (!job) {  in v3d_get_cpu_timestamp_query_params()
457 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_timestamp_query_params()
461 	if (job->job_type) {  in v3d_get_cpu_timestamp_query_params()
462 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_timestamp_query_params()
472 	job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY;  in v3d_get_cpu_timestamp_query_params()
474 	job->timestamp_query.queries = kvmalloc_array(timestamp.count,  in v3d_get_cpu_timestamp_query_params()
477 	if (!job->timestamp_query.queries)  in v3d_get_cpu_timestamp_query_params()
487 			kvfree(job->timestamp_query.queries);  in v3d_get_cpu_timestamp_query_params()
491 		job->timestamp_query.queries[i].offset = offset;  in v3d_get_cpu_timestamp_query_params()
494 			kvfree(job->timestamp_query.queries);  in v3d_get_cpu_timestamp_query_params()
498 		job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync);  in v3d_get_cpu_timestamp_query_params()
500 	job->timestamp_query.count = timestamp.count;  in v3d_get_cpu_timestamp_query_params()
508 				   struct v3d_cpu_job *job)  in v3d_get_cpu_reset_timestamp_params()  argument
513 	if (!job) {  in v3d_get_cpu_reset_timestamp_params()
514 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_reset_timestamp_params()
518 	if (job->job_type) {  in v3d_get_cpu_reset_timestamp_params()
519 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_reset_timestamp_params()
526 	job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY;  in v3d_get_cpu_reset_timestamp_params()
528 	job->timestamp_query.queries = kvmalloc_array(reset.count,  in v3d_get_cpu_reset_timestamp_params()
531 	if (!job->timestamp_query.queries)  in v3d_get_cpu_reset_timestamp_params()
539 		job->timestamp_query.queries[i].offset = reset.offset + 8 * i;  in v3d_get_cpu_reset_timestamp_params()
542 			kvfree(job->timestamp_query.queries);  in v3d_get_cpu_reset_timestamp_params()
546 		job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync);  in v3d_get_cpu_reset_timestamp_params()
548 	job->timestamp_query.count = reset.count;  in v3d_get_cpu_reset_timestamp_params()
553 /* Get data for the copy timestamp query results job submission. */
557 				      struct v3d_cpu_job *job)  in v3d_get_cpu_copy_query_results_params()  argument
563 	if (!job) {  in v3d_get_cpu_copy_query_results_params()
564 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_copy_query_results_params()
568 	if (job->job_type) {  in v3d_get_cpu_copy_query_results_params()
569 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_copy_query_results_params()
579 	job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY;  in v3d_get_cpu_copy_query_results_params()
581 	job->timestamp_query.queries = kvmalloc_array(copy.count,  in v3d_get_cpu_copy_query_results_params()
584 	if (!job->timestamp_query.queries)  in v3d_get_cpu_copy_query_results_params()
594 			kvfree(job->timestamp_query.queries);  in v3d_get_cpu_copy_query_results_params()
598 		job->timestamp_query.queries[i].offset = offset;  in v3d_get_cpu_copy_query_results_params()
601 			kvfree(job->timestamp_query.queries);  in v3d_get_cpu_copy_query_results_params()
605 		job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync);  in v3d_get_cpu_copy_query_results_params()
607 	job->timestamp_query.count = copy.count;  in v3d_get_cpu_copy_query_results_params()
609 	job->copy.do_64bit = copy.do_64bit;  in v3d_get_cpu_copy_query_results_params()
610 	job->copy.do_partial = copy.do_partial;  in v3d_get_cpu_copy_query_results_params()
611 	job->copy.availability_bit = copy.availability_bit;  in v3d_get_cpu_copy_query_results_params()
612 	job->copy.offset = copy.offset;  in v3d_get_cpu_copy_query_results_params()
613 	job->copy.stride = copy.stride;  in v3d_get_cpu_copy_query_results_params()
621 				     struct v3d_cpu_job *job)  in v3d_get_cpu_reset_performance_params()  argument
627 	if (!job) {  in v3d_get_cpu_reset_performance_params()
628 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_reset_performance_params()
632 	if (job->job_type) {  in v3d_get_cpu_reset_performance_params()
633 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_reset_performance_params()
640 	job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY;  in v3d_get_cpu_reset_performance_params()
642 	job->performance_query.queries = kvmalloc_array(reset.count,  in v3d_get_cpu_reset_performance_params()
645 	if (!job->performance_query.queries)  in v3d_get_cpu_reset_performance_params()
658 			kvfree(job->performance_query.queries);  in v3d_get_cpu_reset_performance_params()
662 		job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync);  in v3d_get_cpu_reset_performance_params()
665 			kvfree(job->performance_query.queries);  in v3d_get_cpu_reset_performance_params()
673 				kvfree(job->performance_query.queries);  in v3d_get_cpu_reset_performance_params()
677 			job->performance_query.queries[i].kperfmon_ids[j] = id;  in v3d_get_cpu_reset_performance_params()
680 	job->performance_query.count = reset.count;  in v3d_get_cpu_reset_performance_params()
681 	job->performance_query.nperfmons = reset.nperfmons;  in v3d_get_cpu_reset_performance_params()
689 					  struct v3d_cpu_job *job)  in v3d_get_cpu_copy_performance_query_params()  argument
695 	if (!job) {  in v3d_get_cpu_copy_performance_query_params()
696 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_copy_performance_query_params()
700 	if (job->job_type) {  in v3d_get_cpu_copy_performance_query_params()
701 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_copy_performance_query_params()
711 	job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY;  in v3d_get_cpu_copy_performance_query_params()
713 	job->performance_query.queries = kvmalloc_array(copy.count,  in v3d_get_cpu_copy_performance_query_params()
716 	if (!job->performance_query.queries)  in v3d_get_cpu_copy_performance_query_params()
729 			kvfree(job->performance_query.queries);  in v3d_get_cpu_copy_performance_query_params()
733 		job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync);  in v3d_get_cpu_copy_performance_query_params()
736 			kvfree(job->performance_query.queries);  in v3d_get_cpu_copy_performance_query_params()
744 				kvfree(job->performance_query.queries);  in v3d_get_cpu_copy_performance_query_params()
748 			job->performance_query.queries[i].kperfmon_ids[j] = id;  in v3d_get_cpu_copy_performance_query_params()
751 	job->performance_query.count = copy.count;  in v3d_get_cpu_copy_performance_query_params()
752 	job->performance_query.nperfmons = copy.nperfmons;  in v3d_get_cpu_copy_performance_query_params()
753 	job->performance_query.ncounters = copy.ncounters;  in v3d_get_cpu_copy_performance_query_params()
755 	job->copy.do_64bit = copy.do_64bit;  in v3d_get_cpu_copy_performance_query_params()
756 	job->copy.do_partial = copy.do_partial;  in v3d_get_cpu_copy_performance_query_params()
757 	job->copy.availability_bit = copy.availability_bit;  in v3d_get_cpu_copy_performance_query_params()
758 	job->copy.offset = copy.offset;  in v3d_get_cpu_copy_performance_query_params()
759 	job->copy.stride = copy.stride;  in v3d_get_cpu_copy_performance_query_params()
771 		   struct v3d_cpu_job *job)  in v3d_get_extensions()  argument
790 			ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job);  in v3d_get_extensions()
793 			ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job);  in v3d_get_extensions()
796 			ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job);  in v3d_get_extensions()
799 			ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job);  in v3d_get_extensions()
802 			ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job);  in v3d_get_extensions()
805 			ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job);  in v3d_get_extensions()
822  * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
995  * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
1010 	struct v3d_tfu_job *job = NULL;  in v3d_submit_tfu_ioctl()  local
1029 	ret = v3d_job_allocate((void *)&job, sizeof(*job));  in v3d_submit_tfu_ioctl()
1033 	ret = v3d_job_init(v3d, file_priv, &job->base,  in v3d_submit_tfu_ioctl()
1036 		v3d_job_deallocate((void *)&job);  in v3d_submit_tfu_ioctl()
1040 	job->base.bo = kcalloc(ARRAY_SIZE(args->bo_handles),  in v3d_submit_tfu_ioctl()
1041 			       sizeof(*job->base.bo), GFP_KERNEL);  in v3d_submit_tfu_ioctl()
1042 	if (!job->base.bo) {  in v3d_submit_tfu_ioctl()
1047 	job->args = *args;  in v3d_submit_tfu_ioctl()
1049 	for (job->base.bo_count = 0;  in v3d_submit_tfu_ioctl()
1050 	     job->base.bo_count < ARRAY_SIZE(args->bo_handles);  in v3d_submit_tfu_ioctl()
1051 	     job->base.bo_count++) {  in v3d_submit_tfu_ioctl()
1054 		if (!args->bo_handles[job->base.bo_count])  in v3d_submit_tfu_ioctl()
1057 		bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]);  in v3d_submit_tfu_ioctl()
1060 				  job->base.bo_count,  in v3d_submit_tfu_ioctl()
1061 				  args->bo_handles[job->base.bo_count]);  in v3d_submit_tfu_ioctl()
1065 		job->base.bo[job->base.bo_count] = bo;  in v3d_submit_tfu_ioctl()
1068 	ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx);  in v3d_submit_tfu_ioctl()
1073 	v3d_push_job(&job->base);  in v3d_submit_tfu_ioctl()
1077 						 &job->base, &acquire_ctx,  in v3d_submit_tfu_ioctl()
1080 						 job->base.done_fence);  in v3d_submit_tfu_ioctl()
1082 	v3d_job_put(&job->base);  in v3d_submit_tfu_ioctl()
1087 	v3d_job_cleanup((void *)job);  in v3d_submit_tfu_ioctl()
1094  * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D.
1110 	struct v3d_csd_job *job = NULL;  in v3d_submit_csd_ioctl()  local
1139 					 &job, &clean_job, &se,  in v3d_submit_csd_ioctl()
1145 		job->base.perfmon = v3d_perfmon_find(v3d_priv,  in v3d_submit_csd_ioctl()
1147 		if (!job->base.perfmon) {  in v3d_submit_csd_ioctl()
1154 	v3d_push_job(&job->base);  in v3d_submit_csd_ioctl()
1157 					   dma_fence_get(job->base.done_fence));  in v3d_submit_csd_ioctl()
1171 	v3d_job_put(&job->base);  in v3d_submit_csd_ioctl()
1182 	v3d_job_cleanup((void *)job);  in v3d_submit_csd_ioctl()
1199  * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
1204  * Userspace specifies the CPU job type and data required to perform its
1238 	/* Every CPU job must have a CPU job user extension */  in v3d_submit_cpu_ioctl()
1240 		DRM_DEBUG("CPU job must have a CPU job user extension.\n");  in v3d_submit_cpu_ioctl()
1246 		DRM_DEBUG("This CPU job was not submitted with the proper number of BOs.\n");  in v3d_submit_cpu_ioctl()
1261 	csd_job = cpu_job->indirect_csd.job;  in v3d_submit_cpu_ioctl()