126  * Mark the doorbell as unregistered and reset job queue pointers.
128  * and FW loses job queue state. The next time job queue is used it
171 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)  in ivpu_cmdq_push_job()  argument
173 	struct ivpu_device *vdev = job->vdev;  in ivpu_cmdq_push_job()
179 	/* Check if there is space left in job queue */  in ivpu_cmdq_push_job()
181 		ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",  in ivpu_cmdq_push_job()
182 			 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);  in ivpu_cmdq_push_job()
186 	entry = &cmdq->jobq->job[tail];  in ivpu_cmdq_push_job()
187 	entry->batch_buf_addr = job->cmd_buf_vpu_addr;  in ivpu_cmdq_push_job()
188 	entry->job_id = job->job_id;  in ivpu_cmdq_push_job()
242 static void ivpu_job_destroy(struct ivpu_job *job)  in ivpu_job_destroy()  argument
244 	struct ivpu_device *vdev = job->vdev;  in ivpu_job_destroy()
247 	ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",  in ivpu_job_destroy()
248 		 job->job_id, job->file_priv->ctx.id, job->engine_idx);  in ivpu_job_destroy()
250 	for (i = 0; i < job->bo_count; i++)  in ivpu_job_destroy()
251 		if (job->bos[i])  in ivpu_job_destroy()
252 			drm_gem_object_put(&job->bos[i]->base.base);  in ivpu_job_destroy()
254 	dma_fence_put(job->done_fence);  in ivpu_job_destroy()
255 	ivpu_file_priv_put(&job->file_priv);  in ivpu_job_destroy()
256 	kfree(job);  in ivpu_job_destroy()
263 	struct ivpu_job *job;  in ivpu_job_create()  local
265 	job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);  in ivpu_job_create()
266 	if (!job)  in ivpu_job_create()
269 	job->vdev = vdev;  in ivpu_job_create()
270 	job->engine_idx = engine_idx;  in ivpu_job_create()
271 	job->bo_count = bo_count;  in ivpu_job_create()
272 	job->done_fence = ivpu_fence_create(vdev);  in ivpu_job_create()
273 	if (!job->done_fence) {  in ivpu_job_create()
278 	job->file_priv = ivpu_file_priv_get(file_priv);  in ivpu_job_create()
280 	ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);  in ivpu_job_create()
281 	return job;  in ivpu_job_create()
284 	kfree(job);  in ivpu_job_create()
290 	struct ivpu_job *job;  in ivpu_job_signal_and_destroy()  local
292 	job = xa_erase(&vdev->submitted_jobs_xa, job_id);  in ivpu_job_signal_and_destroy()
293 	if (!job)  in ivpu_job_signal_and_destroy()
296 	if (job->file_priv->has_mmu_faults)  in ivpu_job_signal_and_destroy()
299 	job->bos[CMD_BUF_IDX]->job_status = job_status;  in ivpu_job_signal_and_destroy()
300 	dma_fence_signal(job->done_fence);  in ivpu_job_signal_and_destroy()
302 	ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",  in ivpu_job_signal_and_destroy()
303 		 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);  in ivpu_job_signal_and_destroy()
305 	ivpu_job_destroy(job);  in ivpu_job_signal_and_destroy()
314 	struct ivpu_job *job;  in ivpu_jobs_abort_all()  local
317 	xa_for_each(&vdev->submitted_jobs_xa, id, job)  in ivpu_jobs_abort_all()
321 static int ivpu_job_submit(struct ivpu_job *job)  in ivpu_job_submit()  argument
323 	struct ivpu_file_priv *file_priv = job->file_priv;  in ivpu_job_submit()
324 	struct ivpu_device *vdev = job->vdev;  in ivpu_job_submit()
335 	cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);  in ivpu_job_submit()
337 		ivpu_warn_ratelimited(vdev, "Failed get job queue, ctx %d engine %d\n",  in ivpu_job_submit()
338 				      file_priv->ctx.id, job->engine_idx);  in ivpu_job_submit()
347 	ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);  in ivpu_job_submit()
349 		ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",  in ivpu_job_submit()
355 	ret = ivpu_cmdq_push_job(cmdq, job);  in ivpu_job_submit()
368 	ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d addr 0x%llx next %d\n",  in ivpu_job_submit()
369 		 job->job_id, file_priv->ctx.id, job->engine_idx,  in ivpu_job_submit()
370 		 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);  in ivpu_job_submit()
377 		ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);  in ivpu_job_submit()
382 	__xa_erase(&vdev->submitted_jobs_xa, job->job_id);  in ivpu_job_submit()
392 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,  in ivpu_job_prepare_bos_for_submit()  argument
409 		job->bos[i] = to_ivpu_bo(obj);  in ivpu_job_prepare_bos_for_submit()
411 		ret = ivpu_bo_pin(job->bos[i]);  in ivpu_job_prepare_bos_for_submit()
416 	bo = job->bos[CMD_BUF_IDX];  in ivpu_job_prepare_bos_for_submit()
427 	job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;  in ivpu_job_prepare_bos_for_submit()
429 	ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,  in ivpu_job_prepare_bos_for_submit()
437 		ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);  in ivpu_job_prepare_bos_for_submit()
446 		dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);  in ivpu_job_prepare_bos_for_submit()
450 	drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);  in ivpu_job_prepare_bos_for_submit()
462 	struct ivpu_job *job;  in ivpu_submit_ioctl()  local
501 	ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",  in ivpu_submit_ioctl()
504 	job = ivpu_job_create(file_priv, params->engine, params->buffer_count);  in ivpu_submit_ioctl()
505 	if (!job) {  in ivpu_submit_ioctl()
506 		ivpu_err(vdev, "Failed to create job\n");  in ivpu_submit_ioctl()
511 	ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,  in ivpu_submit_ioctl()
514 		ivpu_err(vdev, "Failed to prepare job: %d\n", ret);  in ivpu_submit_ioctl()
519 	ret = ivpu_job_submit(job);  in ivpu_submit_ioctl()
529 	dma_fence_signal(job->done_fence);  in ivpu_submit_ioctl()
531 	ivpu_job_destroy(job);  in ivpu_submit_ioctl()