accel/ivpu/ivpu_job.c

35 					  struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
44 	cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user,
51 	cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma,
67 					 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
78 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv)
80 	struct ivpu_device *vdev = file_priv->vdev;
92 	ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);
140 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
142 	ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);
147 static struct ivpu_cmdq *ivpu_cmdq_create(struct ivpu_file_priv *file_priv, u8 priority, u32 flags)
149 	struct ivpu_device *vdev = file_priv->vdev;
153 	lockdep_assert_held(&file_priv->lock);
155 	cmdq = ivpu_cmdq_alloc(file_priv);
160 	ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit,
161 			      &file_priv->cmdq_id_next, GFP_KERNEL);
175 		 cmdq->id, file_priv->ctx.id, cmdq->jobq->header.flags);
179 	ivpu_cmdq_free(file_priv, cmdq);
183 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,
186 	struct ivpu_device *vdev = file_priv->vdev;
189 	ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id,
195 	ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id,
203 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
205 	struct ivpu_device *vdev = file_priv->vdev;
216 		ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id,
219 		ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
224 			 cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority);
239 static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
241 	struct ivpu_device *vdev = file_priv->vdev;
244 	lockdep_assert_held(&file_priv->lock);
252 		ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, cmdq->priority);
257 	ret = ivpu_register_db(file_priv, cmdq);
264 static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
266 	struct ivpu_device *vdev = file_priv->vdev;
269 	lockdep_assert_held(&file_priv->lock);
279 		ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id);
282 				 cmdq->id, file_priv->ctx.id);
285 	xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
299 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
301 	ivpu_cmdq_unregister(file_priv, cmdq);
302 	xa_erase(&file_priv->cmdq_xa, cmdq->id);
303 	ivpu_cmdq_free(file_priv, cmdq);
306 static struct ivpu_cmdq *ivpu_cmdq_acquire_legacy(struct ivpu_file_priv *file_priv, u8 priority)
311 	lockdep_assert_held(&file_priv->lock);
313 	xa_for_each(&file_priv->cmdq_xa, id, cmdq)
318 		cmdq = ivpu_cmdq_create(file_priv, priority, 0);
327 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u32 cmdq_id)
329 	struct ivpu_device *vdev = file_priv->vdev;
332 	lockdep_assert_held(&file_priv->lock);
334 	cmdq = xa_load(&file_priv->cmdq_xa, cmdq_id);
343 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
348 	lockdep_assert_held(&file_priv->lock);
350 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)
351 		ivpu_cmdq_destroy(file_priv, cmdq);
360 static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)
365 	mutex_lock(&file_priv->lock);
367 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) {
368 		xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
372 	mutex_unlock(&file_priv->lock);
377 	struct ivpu_file_priv *file_priv;
382 	xa_for_each(&vdev->context_xa, ctx_id, file_priv)
383 		ivpu_cmdq_reset(file_priv);
388 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv)
390 	struct ivpu_device *vdev = file_priv->vdev;
394 	lockdep_assert_held(&file_priv->lock);
395 	ivpu_dbg(vdev, JOB, "Context ID: %u abort\n", file_priv->ctx.id);
397 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)
398 		ivpu_cmdq_unregister(file_priv, cmdq);
401 		ivpu_jsm_context_release(vdev, file_priv->ctx.id);
403 	ivpu_mmu_disable_ssid_events(vdev, file_priv->ctx.id);
405 	file_priv->aborted = true;
419 			 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail);
499 		 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx);
506 	ivpu_file_priv_put(&job->file_priv);
511 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
513 	struct ivpu_device *vdev = file_priv->vdev;
529 	job->file_priv = ivpu_file_priv_get(file_priv);
532 	ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
566 		guard(mutex)(&job->file_priv->lock);
568 		if (job->file_priv->has_mmu_faults)
576 		job->file_priv->has_mmu_faults = true;
585 	if (job->file_priv->has_mmu_faults)
593 		 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx, job_status);
627 		if (job->file_priv->ctx.id == ctx_id && job->cmdq_id == cmdq_id)
635 	struct ivpu_file_priv *file_priv = job->file_priv;
646 	mutex_lock(&file_priv->lock);
649 		cmdq = ivpu_cmdq_acquire_legacy(file_priv, priority);
651 		cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id);
653 		ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d\n", file_priv->ctx.id);
658 	ret = ivpu_cmdq_register(file_priv, cmdq);
667 	ret = xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit,
668 			      &file_priv->job_id_next, GFP_KERNEL);
671 			 file_priv->ctx.id);
693 		 job->job_id, file_priv->ctx.id, cmdq->id, job->engine_idx, cmdq->priority,
696 	mutex_unlock(&file_priv->lock);
709 	mutex_unlock(&file_priv->lock);
719 	struct ivpu_file_priv *file_priv = job->file_priv;
720 	struct ivpu_device *vdev = file_priv->vdev;
781 static int ivpu_submit(struct drm_file *file, struct ivpu_file_priv *file_priv, u32 cmdq_id,
785 	struct ivpu_device *vdev = file_priv->vdev;
806 		 file_priv->ctx.id, cmdq_id, buffer_count);
808 	job = ivpu_job_create(file_priv, engine, buffer_count);
844 	struct ivpu_file_priv *file_priv = file->driver_priv;
860 	if (!file_priv->ctx.id)
863 	if (file_priv->has_mmu_faults)
868 	return ivpu_submit(file, file_priv, 0, args->buffer_count, args->engine,
874 	struct ivpu_file_priv *file_priv = file->driver_priv;
877 	if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ))
889 	if (!file_priv->ctx.id)
892 	if (file_priv->has_mmu_faults)
895 	return ivpu_submit(file, file_priv, args->cmdq_id, args->buffer_count, VPU_ENGINE_COMPUTE,
901 	struct ivpu_file_priv *file_priv = file->driver_priv;
902 	struct ivpu_device *vdev = file_priv->vdev;
917 	mutex_lock(&file_priv->lock);
919 	cmdq = ivpu_cmdq_create(file_priv, ivpu_job_to_jsm_priority(args->priority), args->flags);
923 	mutex_unlock(&file_priv->lock);
932 	struct ivpu_file_priv *file_priv = file->driver_priv;
933 	struct ivpu_device *vdev = file_priv->vdev;
946 	mutex_lock(&file_priv->lock);
948 	cmdq = xa_load(&file_priv->cmdq_xa, args->cmdq_id);
953 		ivpu_cmdq_destroy(file_priv, cmdq);
957 	mutex_unlock(&file_priv->lock);
961 		ivpu_cmdq_abort_all_jobs(vdev, file_priv->ctx.id, cmdq_id);
1005 	struct ivpu_file_priv *file_priv;
1018 	xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
1019 		if (!file_priv->has_mmu_faults || file_priv->aborted)
1022 		mutex_lock(&file_priv->lock);
1023 		ivpu_context_abort_locked(file_priv);
1024 		mutex_unlock(&file_priv->lock);
1047 		if (job->file_priv->aborted)