Lines Matching full:cs
49 /* EBUSY means the CS was never submitted and hence we don't have in hl_fence_release()
59 "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n", in hl_fence_release()
66 * A signal CS can get completion while the corresponding wait in hl_fence_release()
67 * for signal CS is on its way to the PQ. The wait for signal CS in hl_fence_release()
68 * will get stuck if the signal CS incremented the SOB to its in hl_fence_release()
72 * 1. The wait for signal CS must get a ref for the signal CS as in hl_fence_release()
76 * 2. Signal/Wait for signal CS will decrement the SOB refcnt in hl_fence_release()
78 * These two measures guarantee that the wait for signal CS will in hl_fence_release()
79 * reset the SOB upon completion rather than the signal CS and in hl_fence_release()
108 static void cs_get(struct hl_cs *cs) in cs_get() argument
110 kref_get(&cs->refcount); in cs_get()
113 static int cs_get_unless_zero(struct hl_cs *cs) in cs_get_unless_zero() argument
115 return kref_get_unless_zero(&cs->refcount); in cs_get_unless_zero()
118 static void cs_put(struct hl_cs *cs) in cs_put() argument
120 kref_put(&cs->refcount, cs_do_release); in cs_put()
152 parser.ctx_id = job->cs->ctx->asid; in cs_parser()
153 parser.cs_sequence = job->cs->sequence; in cs_parser()
181 * won't be accessed again for this CS in cs_parser()
197 struct hl_cs *cs = job->cs; in free_job() local
232 spin_lock(&cs->job_lock); in free_job()
234 spin_unlock(&cs->job_lock); in free_job()
240 cs_put(cs); in free_job()
261 struct hl_cs *cs = container_of(ref, struct hl_cs, in cs_do_release() local
263 struct hl_device *hdev = cs->ctx->hdev; in cs_do_release()
266 cs->completed = true; in cs_do_release()
270 * finished, because each one of them took refcnt to CS, we still in cs_do_release()
272 * will have leaked memory and what's worse, the CS object (and in cs_do_release()
276 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) in cs_do_release()
280 if (cs->submitted) { in cs_do_release()
293 dev_crit(hdev->dev, "CS active cnt %d is negative\n", in cs_do_release()
299 hl_int_hw_queue_update_ci(cs); in cs_do_release()
302 /* remove CS from hw_queues mirror list */ in cs_do_release()
303 list_del_init(&cs->mirror_node); in cs_do_release()
307 * Don't cancel TDR in case this CS was timedout because we in cs_do_release()
310 if ((!cs->timedout) && in cs_do_release()
314 if (cs->tdr_active) in cs_do_release()
315 cancel_delayed_work_sync(&cs->work_tdr); in cs_do_release()
319 /* queue TDR for next CS */ in cs_do_release()
332 } else if (cs->type == CS_TYPE_WAIT) { in cs_do_release()
334 * In case the wait for signal CS was submitted, the put occurs in cs_do_release()
337 hl_fence_put(cs->signal_fence); in cs_do_release()
344 hl_debugfs_remove_cs(cs); in cs_do_release()
346 hl_ctx_put(cs->ctx); in cs_do_release()
352 if (cs->timedout) in cs_do_release()
353 cs->fence->error = -ETIMEDOUT; in cs_do_release()
354 else if (cs->aborted) in cs_do_release()
355 cs->fence->error = -EIO; in cs_do_release()
356 else if (!cs->submitted) in cs_do_release()
357 cs->fence->error = -EBUSY; in cs_do_release()
359 complete_all(&cs->fence->completion); in cs_do_release()
360 hl_fence_put(cs->fence); in cs_do_release()
361 cs_counters_aggregate(hdev, cs->ctx); in cs_do_release()
363 kfree(cs->jobs_in_queue_cnt); in cs_do_release()
364 kfree(cs); in cs_do_release()
371 struct hl_cs *cs = container_of(work, struct hl_cs, in cs_timedout() local
373 rc = cs_get_unless_zero(cs); in cs_timedout()
377 if ((!cs->submitted) || (cs->completed)) { in cs_timedout()
378 cs_put(cs); in cs_timedout()
382 /* Mark the CS is timed out so we won't try to cancel its TDR */ in cs_timedout()
383 cs->timedout = true; in cs_timedout()
385 hdev = cs->ctx->hdev; in cs_timedout()
389 cs->sequence); in cs_timedout()
391 cs_put(cs); in cs_timedout()
402 struct hl_cs *cs; in allocate_cs() local
405 cs = kzalloc(sizeof(*cs), GFP_ATOMIC); in allocate_cs()
406 if (!cs) in allocate_cs()
409 cs->ctx = ctx; in allocate_cs()
410 cs->submitted = false; in allocate_cs()
411 cs->completed = false; in allocate_cs()
412 cs->type = cs_type; in allocate_cs()
413 INIT_LIST_HEAD(&cs->job_list); in allocate_cs()
414 INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout); in allocate_cs()
415 kref_init(&cs->refcount); in allocate_cs()
416 spin_lock_init(&cs->job_lock); in allocate_cs()
425 cs_cmpl->type = cs->type; in allocate_cs()
427 cs->fence = &cs_cmpl->base_fence; in allocate_cs()
437 "Rejecting CS because of too many in-flights CS\n"); in allocate_cs()
443 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues, in allocate_cs()
444 sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC); in allocate_cs()
445 if (!cs->jobs_in_queue_cnt) { in allocate_cs()
453 cs->sequence = cs_cmpl->cs_seq; in allocate_cs()
466 *cs_new = cs; in allocate_cs()
474 kfree(cs); in allocate_cs()
478 static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs) in cs_rollback() argument
482 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) in cs_rollback()
489 struct hl_cs *cs, *tmp; in hl_cs_rollback_all() local
496 list_for_each_entry_safe(cs, tmp, &hdev->hw_queues_mirror_list, in hl_cs_rollback_all()
498 cs_get(cs); in hl_cs_rollback_all()
499 cs->aborted = true; in hl_cs_rollback_all()
500 dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n", in hl_cs_rollback_all()
501 cs->ctx->asid, cs->sequence); in hl_cs_rollback_all()
502 cs_rollback(hdev, cs); in hl_cs_rollback_all()
503 cs_put(cs); in hl_cs_rollback_all()
511 struct hl_cs *cs = job->cs; in job_wq_completion() local
512 struct hl_device *hdev = cs->ctx->hdev; in job_wq_completion()
614 struct hl_cs *cs; in cs_ioctl_default() local
639 dev_err(hdev->dev, "Failed to copy cs chunk array from user\n"); in cs_ioctl_default()
647 rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs); in cs_ioctl_default()
653 *cs_seq = cs->sequence; in cs_ioctl_default()
655 hl_debugfs_add_cs(cs); in cs_ioctl_default()
657 /* Validate ALL the CS chunks before submitting the CS */ in cs_ioctl_default()
697 job->cs = cs; in cs_ioctl_default()
702 cs->jobs_in_queue_cnt[job->hw_queue_id]++; in cs_ioctl_default()
704 list_add_tail(&job->cs_node, &cs->job_list); in cs_ioctl_default()
707 * Increment CS reference. When CS reference is 0, CS is in cs_ioctl_default()
714 cs_get(cs); in cs_ioctl_default()
722 "Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n", in cs_ioctl_default()
723 cs->ctx->asid, cs->sequence, job->id, rc); in cs_ioctl_default()
731 "Reject CS %d.%llu because only internal queues jobs are present\n", in cs_ioctl_default()
732 cs->ctx->asid, cs->sequence); in cs_ioctl_default()
737 rc = hl_hw_queue_schedule_cs(cs); in cs_ioctl_default()
741 "Failed to submit CS %d.%llu to H/W queues, error %d\n", in cs_ioctl_default()
742 cs->ctx->asid, cs->sequence, rc); in cs_ioctl_default()
755 cs_rollback(hdev, cs); in cs_ioctl_default()
759 /* We finished with the CS in this function, so put the ref */ in cs_ioctl_default()
760 cs_put(cs); in cs_ioctl_default()
777 struct hl_cs *cs; in cs_ioctl_signal_wait() local
803 dev_err(hdev->dev, "Failed to copy cs chunk array from user\n"); in cs_ioctl_signal_wait()
837 "Wait for signal CS supports only one signal CS seq\n"); in cs_ioctl_signal_wait()
866 "Failed to get signal CS with seq 0x%llx\n", in cs_ioctl_signal_wait()
873 /* signal CS already finished */ in cs_ioctl_signal_wait()
883 "CS seq 0x%llx is not of a signal CS\n", in cs_ioctl_signal_wait()
891 /* signal CS already finished */ in cs_ioctl_signal_wait()
901 rc = allocate_cs(hdev, ctx, cs_type, &cs); in cs_ioctl_signal_wait()
910 * Save the signal CS fence for later initialization right before in cs_ioctl_signal_wait()
911 * hanging the wait CS on the queue. in cs_ioctl_signal_wait()
913 if (cs->type == CS_TYPE_WAIT) in cs_ioctl_signal_wait()
914 cs->signal_fence = sig_fence; in cs_ioctl_signal_wait()
916 hl_debugfs_add_cs(cs); in cs_ioctl_signal_wait()
918 *cs_seq = cs->sequence; in cs_ioctl_signal_wait()
928 if (cs->type == CS_TYPE_WAIT) in cs_ioctl_signal_wait()
943 job->cs = cs; in cs_ioctl_signal_wait()
959 cs->jobs_in_queue_cnt[job->hw_queue_id]++; in cs_ioctl_signal_wait()
961 list_add_tail(&job->cs_node, &cs->job_list); in cs_ioctl_signal_wait()
964 cs_get(cs); in cs_ioctl_signal_wait()
968 rc = hl_hw_queue_schedule_cs(cs); in cs_ioctl_signal_wait()
972 "Failed to submit CS %d.%llu to H/W queues, error %d\n", in cs_ioctl_signal_wait()
973 ctx->asid, cs->sequence, rc); in cs_ioctl_signal_wait()
981 cs_rollback(hdev, cs); in cs_ioctl_signal_wait()
985 /* We finished with the CS in this function, so put the ref */ in cs_ioctl_signal_wait()
986 cs_put(cs); in cs_ioctl_signal_wait()
1010 "Device is %s. Can't submit new CS\n", in hl_cs_ioctl()
1020 "Signal and wait CS flags are mutually exclusive, context %d\n", in hl_cs_ioctl()
1028 dev_err(hdev->dev, "Sync stream CS is not supported\n"); in hl_cs_ioctl()
1046 "Got execute CS with 0 chunks, context %d\n", in hl_cs_ioctl()
1053 "Sync stream CS mandates one chunk only, context %d\n", in hl_cs_ioctl()
1074 "Failed to switch to context %d, rejecting CS! %d\n", in hl_cs_ioctl()
1096 "Need to run restore phase but restore CS is empty\n"); in hl_cs_ioctl()
1107 "Failed to submit restore CS for context %d (%d)\n", in hl_cs_ioctl()
1119 "Restore CS for context %d failed to complete %ld\n", in hl_cs_ioctl()
1180 "Can't wait on CS %llu because current CS is at seq %llu\n", in _hl_cs_wait_ioctl()
1197 "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n", in _hl_cs_wait_ioctl()
1221 "user process got signal while waiting for CS handle %llu\n", in hl_cs_wait_ioctl()
1227 "CS %llu has timed-out while user process is waiting for it\n", in hl_cs_wait_ioctl()
1232 "CS %llu has been aborted while user process is waiting for it\n", in hl_cs_wait_ioctl()