Lines Matching refs:q
110 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
113 static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) in xe_exec_queue_group_cleanup() argument
115 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_cleanup()
119 if (xe_exec_queue_is_multi_queue_secondary(q)) { in xe_exec_queue_group_cleanup()
124 xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q)); in xe_exec_queue_group_cleanup()
141 static void __xe_exec_queue_free(struct xe_exec_queue *q) in __xe_exec_queue_free() argument
146 if (q->tlb_inval[i].dep_scheduler) in __xe_exec_queue_free()
147 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); in __xe_exec_queue_free()
149 if (xe_exec_queue_uses_pxp(q)) in __xe_exec_queue_free()
150 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); in __xe_exec_queue_free()
152 if (xe_exec_queue_is_multi_queue(q)) in __xe_exec_queue_free()
153 xe_exec_queue_group_cleanup(q); in __xe_exec_queue_free()
155 if (q->vm) in __xe_exec_queue_free()
156 xe_vm_put(q->vm); in __xe_exec_queue_free()
158 if (q->xef) in __xe_exec_queue_free()
159 xe_file_put(q->xef); in __xe_exec_queue_free()
161 kvfree(q->replay_state); in __xe_exec_queue_free()
162 kfree(q); in __xe_exec_queue_free()
165 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) in alloc_dep_schedulers() argument
167 struct xe_tile *tile = gt_to_tile(q->gt); in alloc_dep_schedulers()
186 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, in alloc_dep_schedulers()
191 q->tlb_inval[i].dep_scheduler = dep_scheduler; in alloc_dep_schedulers()
204 struct xe_exec_queue *q; in __xe_exec_queue_alloc() local
211 q = kzalloc_flex(*q, lrc, width); in __xe_exec_queue_alloc()
212 if (!q) in __xe_exec_queue_alloc()
215 kref_init(&q->refcount); in __xe_exec_queue_alloc()
216 q->flags = flags; in __xe_exec_queue_alloc()
217 q->hwe = hwe; in __xe_exec_queue_alloc()
218 q->gt = gt; in __xe_exec_queue_alloc()
219 q->class = hwe->class; in __xe_exec_queue_alloc()
220 q->width = width; in __xe_exec_queue_alloc()
221 q->msix_vec = XE_IRQ_DEFAULT_MSIX; in __xe_exec_queue_alloc()
222 q->logical_mask = logical_mask; in __xe_exec_queue_alloc()
223 q->fence_irq = >->fence_irq[hwe->class]; in __xe_exec_queue_alloc()
224 q->ring_ops = gt->ring_ops[hwe->class]; in __xe_exec_queue_alloc()
225 q->ops = gt->exec_queue_ops; in __xe_exec_queue_alloc()
226 INIT_LIST_HEAD(&q->lr.link); in __xe_exec_queue_alloc()
227 INIT_LIST_HEAD(&q->multi_gt_link); in __xe_exec_queue_alloc()
228 INIT_LIST_HEAD(&q->hw_engine_group_link); in __xe_exec_queue_alloc()
229 INIT_LIST_HEAD(&q->pxp.link); in __xe_exec_queue_alloc()
230 q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; in __xe_exec_queue_alloc()
232 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; in __xe_exec_queue_alloc()
233 q->sched_props.preempt_timeout_us = in __xe_exec_queue_alloc()
235 q->sched_props.job_timeout_ms = in __xe_exec_queue_alloc()
237 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && in __xe_exec_queue_alloc()
238 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) in __xe_exec_queue_alloc()
239 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; in __xe_exec_queue_alloc()
241 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; in __xe_exec_queue_alloc()
243 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { in __xe_exec_queue_alloc()
244 err = alloc_dep_schedulers(xe, q); in __xe_exec_queue_alloc()
246 __xe_exec_queue_free(q); in __xe_exec_queue_alloc()
252 q->vm = xe_vm_get(vm); in __xe_exec_queue_alloc()
259 err = exec_queue_user_extensions(xe, q, extensions); in __xe_exec_queue_alloc()
261 __xe_exec_queue_free(q); in __xe_exec_queue_alloc()
266 return q; in __xe_exec_queue_alloc()
269 static void __xe_exec_queue_fini(struct xe_exec_queue *q) in __xe_exec_queue_fini() argument
273 q->ops->fini(q); in __xe_exec_queue_fini()
275 for (i = 0; i < q->width; ++i) in __xe_exec_queue_fini()
276 xe_lrc_put(q->lrc[i]); in __xe_exec_queue_fini()
279 static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) in __xe_exec_queue_init() argument
290 if (xe_exec_queue_uses_pxp(q) && in __xe_exec_queue_init()
291 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { in __xe_exec_queue_init()
292 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) in __xe_exec_queue_init()
301 err = q->ops->init(q); in __xe_exec_queue_init()
315 for (i = 0; i < q->width; ++i) { in __xe_exec_queue_init()
318 xe_gt_sriov_vf_wait_valid_ggtt(q->gt); in __xe_exec_queue_init()
319 lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, in __xe_exec_queue_init()
320 xe_lrc_ring_size(), q->msix_vec, flags); in __xe_exec_queue_init()
327 WRITE_ONCE(q->lrc[i], lrc); in __xe_exec_queue_init()
333 __xe_exec_queue_fini(q); in __xe_exec_queue_init()
342 struct xe_exec_queue *q; in xe_exec_queue_create() local
348 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, in xe_exec_queue_create()
350 if (IS_ERR(q)) in xe_exec_queue_create()
351 return q; in xe_exec_queue_create()
353 err = __xe_exec_queue_init(q, flags); in xe_exec_queue_create()
364 if (xe_exec_queue_uses_pxp(q)) { in xe_exec_queue_create()
365 err = xe_pxp_exec_queue_add(xe->pxp, q); in xe_exec_queue_create()
370 return q; in xe_exec_queue_create()
373 __xe_exec_queue_fini(q); in xe_exec_queue_create()
375 __xe_exec_queue_free(q); in xe_exec_queue_create()
429 struct xe_exec_queue *q; in xe_exec_queue_create_bind() local
444 q = xe_exec_queue_create(xe, migrate_vm, in xe_exec_queue_create_bind()
448 q = xe_exec_queue_create_class(xe, gt, migrate_vm, in xe_exec_queue_create_bind()
454 if (!IS_ERR(q)) { in xe_exec_queue_create_bind()
455 int err = drm_syncobj_create(&q->ufence_syncobj, in xe_exec_queue_create_bind()
459 xe_exec_queue_put(q); in xe_exec_queue_create_bind()
464 q->user_vm = xe_vm_get(user_vm); in xe_exec_queue_create_bind()
467 return q; in xe_exec_queue_create_bind()
473 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); in xe_exec_queue_destroy() local
477 xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); in xe_exec_queue_destroy()
479 if (q->ufence_syncobj) in xe_exec_queue_destroy()
480 drm_syncobj_put(q->ufence_syncobj); in xe_exec_queue_destroy()
482 if (xe_exec_queue_uses_pxp(q)) in xe_exec_queue_destroy()
483 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); in xe_exec_queue_destroy()
485 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_destroy()
487 xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i); in xe_exec_queue_destroy()
489 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { in xe_exec_queue_destroy()
490 list_for_each_entry_safe(eq, next, &q->multi_gt_list, in xe_exec_queue_destroy()
495 if (q->user_vm) { in xe_exec_queue_destroy()
496 xe_vm_put(q->user_vm); in xe_exec_queue_destroy()
497 q->user_vm = NULL; in xe_exec_queue_destroy()
500 q->ops->destroy(q); in xe_exec_queue_destroy()
503 void xe_exec_queue_fini(struct xe_exec_queue *q) in xe_exec_queue_fini() argument
509 xe_exec_queue_update_run_ticks(q); in xe_exec_queue_fini()
510 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) in xe_exec_queue_fini()
511 wake_up_var(&q->xef->exec_queue.pending_removal); in xe_exec_queue_fini()
513 __xe_exec_queue_fini(q); in xe_exec_queue_fini()
514 __xe_exec_queue_free(q); in xe_exec_queue_fini()
517 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) in xe_exec_queue_assign_name() argument
519 switch (q->class) { in xe_exec_queue_assign_name()
521 snprintf(q->name, sizeof(q->name), "rcs%d", instance); in xe_exec_queue_assign_name()
524 snprintf(q->name, sizeof(q->name), "vcs%d", instance); in xe_exec_queue_assign_name()
527 snprintf(q->name, sizeof(q->name), "vecs%d", instance); in xe_exec_queue_assign_name()
530 snprintf(q->name, sizeof(q->name), "bcs%d", instance); in xe_exec_queue_assign_name()
533 snprintf(q->name, sizeof(q->name), "ccs%d", instance); in xe_exec_queue_assign_name()
536 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); in xe_exec_queue_assign_name()
539 XE_WARN_ON(q->class); in xe_exec_queue_assign_name()
545 struct xe_exec_queue *q; in xe_exec_queue_lookup() local
548 q = xa_load(&xef->exec_queue.xa, id); in xe_exec_queue_lookup()
549 if (q) in xe_exec_queue_lookup()
550 xe_exec_queue_get(q); in xe_exec_queue_lookup()
553 return q; in xe_exec_queue_lookup()
563 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_priority() argument
572 q->sched_props.priority = value; in exec_queue_set_priority()
628 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_timeslice() argument
633 xe_exec_queue_get_prop_minmax(q->hwe->eclass, in exec_queue_set_timeslice()
640 q->sched_props.timeslice_us = value; in exec_queue_set_timeslice()
645 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) in exec_queue_set_pxp_type() argument
657 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); in exec_queue_set_pxp_type()
661 struct xe_exec_queue *q, in exec_queue_set_hang_replay_state() argument
664 size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class); in exec_queue_set_hang_replay_state()
672 q->replay_state = ptr; in exec_queue_set_hang_replay_state()
677 static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q) in xe_exec_queue_group_init() argument
679 struct xe_tile *tile = gt_to_tile(q->gt); in xe_exec_queue_group_init()
702 group->primary = q; in xe_exec_queue_group_init()
707 q->multi_queue.group = group; in xe_exec_queue_group_init()
719 static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q) in xe_exec_queue_supports_multi_queue() argument
721 return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); in xe_exec_queue_supports_multi_queue()
724 static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, in xe_exec_queue_group_validate() argument
735 primary = xe_exec_queue_lookup(q->vm->xef, primary_id); in xe_exec_queue_group_validate()
740 XE_IOCTL_DBG(xe, q->vm != primary->vm) || in xe_exec_queue_group_validate()
741 XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) { in xe_exec_queue_group_validate()
747 q->multi_queue.valid = true; in xe_exec_queue_group_validate()
748 q->multi_queue.group = group; in xe_exec_queue_group_validate()
757 static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q) in xe_exec_queue_group_add() argument
759 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_add()
763 xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); in xe_exec_queue_group_add()
766 err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]), in xe_exec_queue_group_add()
769 xe_lrc_put(q->lrc[0]); in xe_exec_queue_group_add()
778 q->multi_queue.pos = pos; in xe_exec_queue_group_add()
783 static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q) in xe_exec_queue_group_delete() argument
785 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_delete()
788 xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); in xe_exec_queue_group_delete()
790 lrc = xa_erase(&group->xa, q->multi_queue.pos); in xe_exec_queue_group_delete()
795 static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_multi_group() argument
798 if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q))) in exec_queue_set_multi_group()
804 if (XE_IOCTL_DBG(xe, !q->vm->xef)) in exec_queue_set_multi_group()
807 if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q))) in exec_queue_set_multi_group()
810 if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q))) in exec_queue_set_multi_group()
817 q->multi_queue.valid = true; in exec_queue_set_multi_group()
818 q->multi_queue.is_primary = true; in exec_queue_set_multi_group()
819 q->multi_queue.pos = 0; in exec_queue_set_multi_group()
827 return xe_exec_queue_group_validate(xe, q, value); in exec_queue_set_multi_group()
830 static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_multi_queue_priority() argument
837 if (!q->xef) { in exec_queue_set_multi_queue_priority()
838 q->multi_queue.priority = value; in exec_queue_set_multi_queue_priority()
842 if (!xe_exec_queue_is_multi_queue(q)) in exec_queue_set_multi_queue_priority()
845 return q->ops->set_multi_queue_priority(q, value); in exec_queue_set_multi_queue_priority()
849 struct xe_exec_queue *q,
868 struct xe_exec_queue *q; in xe_exec_queue_set_property_ioctl() local
879 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_exec_queue_set_property_ioctl()
880 if (XE_IOCTL_DBG(xe, !q)) in xe_exec_queue_set_property_ioctl()
885 ret = exec_queue_set_property_funcs[idx](xe, q, args->value); in xe_exec_queue_set_property_ioctl()
889 xe_exec_queue_put(q); in xe_exec_queue_set_property_ioctl()
893 xe_exec_queue_put(q); in xe_exec_queue_set_property_ioctl()
897 static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties) in exec_queue_user_ext_check() argument
906 if (xe_exec_queue_is_multi_queue_secondary(q) && in exec_queue_user_ext_check()
913 static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties) in exec_queue_user_ext_check_final() argument
924 struct xe_exec_queue *q, in exec_queue_user_ext_set_property() argument
952 err = exec_queue_user_ext_check(q, *properties); in exec_queue_user_ext_set_property()
956 return exec_queue_set_property_funcs[idx](xe, q, ext.value); in exec_queue_user_ext_set_property()
960 struct xe_exec_queue *q,
968 static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, in __exec_queue_user_extensions() argument
990 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties); in __exec_queue_user_extensions()
995 return __exec_queue_user_extensions(xe, q, ext.next_extension, in __exec_queue_user_extensions()
1001 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_user_extensions() argument
1007 err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties); in exec_queue_user_extensions()
1011 err = exec_queue_user_ext_check_final(q, properties); in exec_queue_user_extensions()
1015 if (xe_exec_queue_is_multi_queue_primary(q)) { in exec_queue_user_extensions()
1016 err = xe_exec_queue_group_init(xe, q); in exec_queue_user_extensions()
1098 struct xe_exec_queue *q = NULL; in xe_exec_queue_create_ioctl() local
1159 if (q) in xe_exec_queue_create_ioctl()
1164 q = new; in xe_exec_queue_create_ioctl()
1167 &q->multi_gt_link); in xe_exec_queue_create_ioctl()
1205 q = xe_exec_queue_create(xe, vm, logical_mask, in xe_exec_queue_create_ioctl()
1210 if (IS_ERR(q)) in xe_exec_queue_create_ioctl()
1211 return PTR_ERR(q); in xe_exec_queue_create_ioctl()
1213 if (xe_exec_queue_is_multi_queue_secondary(q)) { in xe_exec_queue_create_ioctl()
1214 err = xe_exec_queue_group_add(xe, q); in xe_exec_queue_create_ioctl()
1220 q->lr.context = dma_fence_context_alloc(1); in xe_exec_queue_create_ioctl()
1222 err = xe_vm_add_compute_exec_queue(vm, q); in xe_exec_queue_create_ioctl()
1227 if (q->vm && q->hwe->hw_engine_group) { in xe_exec_queue_create_ioctl()
1228 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_create_ioctl()
1234 q->xef = xe_file_get(xef); in xe_exec_queue_create_ioctl()
1237 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); in xe_exec_queue_create_ioctl()
1246 xe_exec_queue_kill(q); in xe_exec_queue_create_ioctl()
1248 if (xe_exec_queue_is_multi_queue_secondary(q)) in xe_exec_queue_create_ioctl()
1249 xe_exec_queue_group_delete(xe, q); in xe_exec_queue_create_ioctl()
1251 xe_exec_queue_put(q); in xe_exec_queue_create_ioctl()
1261 struct xe_exec_queue *q; in xe_exec_queue_get_property_ioctl() local
1267 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_exec_queue_get_property_ioctl()
1268 if (XE_IOCTL_DBG(xe, !q)) in xe_exec_queue_get_property_ioctl()
1273 args->value = q->ops->reset_status(q); in xe_exec_queue_get_property_ioctl()
1280 xe_exec_queue_put(q); in xe_exec_queue_get_property_ioctl()
1295 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) in xe_exec_queue_lrc() argument
1297 return q->lrc[0]; in xe_exec_queue_lrc()
1306 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) in xe_exec_queue_is_lr() argument
1308 return q->vm && xe_vm_in_lr_mode(q->vm) && in xe_exec_queue_is_lr()
1309 !(q->flags & EXEC_QUEUE_FLAG_VM); in xe_exec_queue_is_lr()
1326 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) in xe_exec_queue_is_idle() argument
1328 if (xe_exec_queue_is_parallel(q)) { in xe_exec_queue_is_idle()
1331 for (i = 0; i < q->width; ++i) { in xe_exec_queue_is_idle()
1332 if (xe_lrc_seqno(q->lrc[i]) != in xe_exec_queue_is_idle()
1333 q->lrc[i]->fence_ctx.next_seqno - 1) in xe_exec_queue_is_idle()
1340 return xe_lrc_seqno(q->lrc[0]) == in xe_exec_queue_is_idle()
1341 q->lrc[0]->fence_ctx.next_seqno - 1; in xe_exec_queue_is_idle()
1352 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) in xe_exec_queue_update_run_ticks() argument
1354 struct xe_device *xe = gt_to_xe(q->gt); in xe_exec_queue_update_run_ticks()
1363 if (!q->xef) in xe_exec_queue_update_run_ticks()
1377 lrc = q->lrc[0]; in xe_exec_queue_update_run_ticks()
1379 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; in xe_exec_queue_update_run_ticks()
1393 void xe_exec_queue_kill(struct xe_exec_queue *q) in xe_exec_queue_kill() argument
1395 struct xe_exec_queue *eq = q, *next; in xe_exec_queue_kill()
1399 q->ops->kill(eq); in xe_exec_queue_kill()
1400 xe_vm_remove_compute_exec_queue(q->vm, eq); in xe_exec_queue_kill()
1403 q->ops->kill(q); in xe_exec_queue_kill()
1404 xe_vm_remove_compute_exec_queue(q->vm, q); in xe_exec_queue_kill()
1413 struct xe_exec_queue *q; in xe_exec_queue_destroy_ioctl() local
1420 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); in xe_exec_queue_destroy_ioctl()
1421 if (q) in xe_exec_queue_destroy_ioctl()
1425 if (XE_IOCTL_DBG(xe, !q)) in xe_exec_queue_destroy_ioctl()
1428 if (q->vm && q->hwe->hw_engine_group) in xe_exec_queue_destroy_ioctl()
1429 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_destroy_ioctl()
1431 xe_exec_queue_kill(q); in xe_exec_queue_destroy_ioctl()
1433 trace_xe_exec_queue_close(q); in xe_exec_queue_destroy_ioctl()
1434 xe_exec_queue_put(q); in xe_exec_queue_destroy_ioctl()
1439 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, in xe_exec_queue_last_fence_lockdep_assert() argument
1442 if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) { in xe_exec_queue_last_fence_lockdep_assert()
1443 xe_migrate_job_lock_assert(q); in xe_exec_queue_last_fence_lockdep_assert()
1444 } else if (q->flags & EXEC_QUEUE_FLAG_VM) { in xe_exec_queue_last_fence_lockdep_assert()
1448 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_lockdep_assert()
1457 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) in xe_exec_queue_last_fence_put() argument
1459 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_put()
1461 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_last_fence_put()
1470 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) in xe_exec_queue_last_fence_put_unlocked() argument
1472 if (q->last_fence) { in xe_exec_queue_last_fence_put_unlocked()
1473 dma_fence_put(q->last_fence); in xe_exec_queue_last_fence_put_unlocked()
1474 q->last_fence = NULL; in xe_exec_queue_last_fence_put_unlocked()
1487 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, in xe_exec_queue_last_fence_get() argument
1492 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_get()
1494 if (q->last_fence && in xe_exec_queue_last_fence_get()
1495 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get()
1496 xe_exec_queue_last_fence_put(q, vm); in xe_exec_queue_last_fence_get()
1498 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get()
1514 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, in xe_exec_queue_last_fence_get_for_resume() argument
1519 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_get_for_resume()
1521 if (q->last_fence && in xe_exec_queue_last_fence_get_for_resume()
1522 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get_for_resume()
1523 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_last_fence_get_for_resume()
1525 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get_for_resume()
1539 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, in xe_exec_queue_last_fence_set() argument
1542 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_set()
1545 xe_exec_queue_last_fence_put(q, vm); in xe_exec_queue_last_fence_set()
1546 q->last_fence = dma_fence_get(fence); in xe_exec_queue_last_fence_set()
1555 void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, in xe_exec_queue_tlb_inval_last_fence_put() argument
1559 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_tlb_inval_last_fence_put()
1563 xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type); in xe_exec_queue_tlb_inval_last_fence_put()
1574 void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, in xe_exec_queue_tlb_inval_last_fence_put_unlocked() argument
1577 xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1580 dma_fence_put(q->tlb_inval[type].last_fence); in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1581 q->tlb_inval[type].last_fence = NULL; in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1594 struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q, in xe_exec_queue_tlb_inval_last_fence_get() argument
1600 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_tlb_inval_last_fence_get()
1603 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | in xe_exec_queue_tlb_inval_last_fence_get()
1606 if (q->tlb_inval[type].last_fence && in xe_exec_queue_tlb_inval_last_fence_get()
1608 &q->tlb_inval[type].last_fence->flags)) in xe_exec_queue_tlb_inval_last_fence_get()
1609 xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); in xe_exec_queue_tlb_inval_last_fence_get()
1611 fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub(); in xe_exec_queue_tlb_inval_last_fence_get()
1627 void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q, in xe_exec_queue_tlb_inval_last_fence_set() argument
1632 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_tlb_inval_last_fence_set()
1635 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | in xe_exec_queue_tlb_inval_last_fence_set()
1639 xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); in xe_exec_queue_tlb_inval_last_fence_set()
1640 q->tlb_inval[type].last_fence = dma_fence_get(fence); in xe_exec_queue_tlb_inval_last_fence_set()
1651 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) in xe_exec_queue_contexts_hwsp_rebase() argument
1656 for (i = 0; i < q->width; ++i) { in xe_exec_queue_contexts_hwsp_rebase()
1660 lrc = READ_ONCE(q->lrc[i]); in xe_exec_queue_contexts_hwsp_rebase()
1664 xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch); in xe_exec_queue_contexts_hwsp_rebase()
1666 err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch); in xe_exec_queue_contexts_hwsp_rebase()