Lines Matching +full:cs +full:- +full:0
1 // SPDX-License-Identifier: MIT
16 u32 *cs, flags = 0; in gen8_emit_flush_rcs() local
42 if (GRAPHICS_VER(rq->i915) == 9) in gen8_emit_flush_rcs()
46 if (IS_KABYLAKE(rq->i915) && IS_GRAPHICS_STEP(rq->i915, 0, STEP_C0)) in gen8_emit_flush_rcs()
58 cs = intel_ring_begin(rq, len); in gen8_emit_flush_rcs()
59 if (IS_ERR(cs)) in gen8_emit_flush_rcs()
60 return PTR_ERR(cs); in gen8_emit_flush_rcs()
63 cs = gen8_emit_pipe_control(cs, 0, 0); in gen8_emit_flush_rcs()
66 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, in gen8_emit_flush_rcs()
67 0); in gen8_emit_flush_rcs()
69 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen8_emit_flush_rcs()
72 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); in gen8_emit_flush_rcs()
74 intel_ring_advance(rq, cs); in gen8_emit_flush_rcs()
76 return 0; in gen8_emit_flush_rcs()
81 u32 cmd, *cs; in gen8_emit_flush_xcs() local
83 cs = intel_ring_begin(rq, 4); in gen8_emit_flush_xcs()
84 if (IS_ERR(cs)) in gen8_emit_flush_xcs()
85 return PTR_ERR(cs); in gen8_emit_flush_xcs()
99 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen8_emit_flush_xcs()
103 *cs++ = cmd; in gen8_emit_flush_xcs()
104 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen8_emit_flush_xcs()
105 *cs++ = 0; /* upper addr */ in gen8_emit_flush_xcs()
106 *cs++ = 0; /* value */ in gen8_emit_flush_xcs()
107 intel_ring_advance(rq, cs); in gen8_emit_flush_xcs()
109 return 0; in gen8_emit_flush_xcs()
115 u32 *cs; in gen11_emit_flush_rcs() local
116 u32 flags = 0; in gen11_emit_flush_rcs()
128 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
129 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
130 return PTR_ERR(cs); in gen11_emit_flush_rcs()
132 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
133 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
137 u32 *cs; in gen11_emit_flush_rcs() local
138 u32 flags = 0; in gen11_emit_flush_rcs()
152 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
153 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
154 return PTR_ERR(cs); in gen11_emit_flush_rcs()
156 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
157 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
160 return 0; in gen11_emit_flush_rcs()
170 switch (engine->id) { in gen12_get_aux_inv_reg()
192 if (IS_PONTEVECCHIO(engine->i915)) in gen12_needs_ccs_aux_inv()
199 return i915_mmio_reg_valid(reg) && !HAS_FLAT_CCS(engine->i915); in gen12_needs_ccs_aux_inv()
202 u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs) in gen12_emit_aux_table_inv() argument
205 u32 gsi_offset = engine->gt->uncore->gsi_offset; in gen12_emit_aux_table_inv()
208 return cs; in gen12_emit_aux_table_inv()
210 *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; in gen12_emit_aux_table_inv()
211 *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; in gen12_emit_aux_table_inv()
212 *cs++ = AUX_INV; in gen12_emit_aux_table_inv()
214 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_aux_table_inv()
218 *cs++ = 0; in gen12_emit_aux_table_inv()
219 *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; in gen12_emit_aux_table_inv()
220 *cs++ = 0; in gen12_emit_aux_table_inv()
221 *cs++ = 0; in gen12_emit_aux_table_inv()
223 return cs; in gen12_emit_aux_table_inv()
229 if (IS_GFX_GT_IP_RANGE(rq->engine->gt, IP_VER(12, 70), IP_VER(12, 71)) || in mtl_dummy_pipe_control()
230 IS_DG2(rq->i915)) { in mtl_dummy_pipe_control()
231 u32 *cs; in mtl_dummy_pipe_control() local
234 cs = intel_ring_begin(rq, 6); in mtl_dummy_pipe_control()
235 if (IS_ERR(cs)) in mtl_dummy_pipe_control()
236 return PTR_ERR(cs); in mtl_dummy_pipe_control()
237 cs = gen12_emit_pipe_control(cs, in mtl_dummy_pipe_control()
238 0, in mtl_dummy_pipe_control()
241 intel_ring_advance(rq, cs); in mtl_dummy_pipe_control()
244 return 0; in mtl_dummy_pipe_control()
249 struct intel_engine_cs *engine = rq->engine; in gen12_emit_flush_rcs()
256 u32 bit_group_0 = 0; in gen12_emit_flush_rcs()
257 u32 bit_group_1 = 0; in gen12_emit_flush_rcs()
259 u32 *cs; in gen12_emit_flush_rcs() local
271 if (GRAPHICS_VER_FULL(rq->i915) >= IP_VER(12, 70)) in gen12_emit_flush_rcs()
276 * which happens as part of pipe-control so we can in gen12_emit_flush_rcs()
282 GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70)) in gen12_emit_flush_rcs()
288 /* Wa_1409600907:tgl,adl-p */ in gen12_emit_flush_rcs()
298 if (!HAS_3D_PIPELINE(engine->i915)) in gen12_emit_flush_rcs()
300 else if (engine->class == COMPUTE_CLASS) in gen12_emit_flush_rcs()
303 cs = intel_ring_begin(rq, 6); in gen12_emit_flush_rcs()
304 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
305 return PTR_ERR(cs); in gen12_emit_flush_rcs()
307 cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1, in gen12_emit_flush_rcs()
309 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
313 u32 flags = 0; in gen12_emit_flush_rcs()
314 u32 *cs, count; in gen12_emit_flush_rcs() local
334 if (!HAS_3D_PIPELINE(engine->i915)) in gen12_emit_flush_rcs()
336 else if (engine->class == COMPUTE_CLASS) in gen12_emit_flush_rcs()
340 if (gen12_needs_ccs_aux_inv(rq->engine)) in gen12_emit_flush_rcs()
343 cs = intel_ring_begin(rq, count); in gen12_emit_flush_rcs()
344 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
345 return PTR_ERR(cs); in gen12_emit_flush_rcs()
348 * Prevent the pre-parser from skipping past the TLB in gen12_emit_flush_rcs()
352 *cs++ = preparser_disable(true); in gen12_emit_flush_rcs()
354 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen12_emit_flush_rcs()
356 cs = gen12_emit_aux_table_inv(engine, cs); in gen12_emit_flush_rcs()
358 *cs++ = preparser_disable(false); in gen12_emit_flush_rcs()
359 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
362 return 0; in gen12_emit_flush_rcs()
368 u32 *cs; in gen12_emit_flush_xcs() local
373 if (gen12_needs_ccs_aux_inv(rq->engine)) in gen12_emit_flush_xcs()
377 cs = intel_ring_begin(rq, cmd); in gen12_emit_flush_xcs()
378 if (IS_ERR(cs)) in gen12_emit_flush_xcs()
379 return PTR_ERR(cs); in gen12_emit_flush_xcs()
382 *cs++ = preparser_disable(true); in gen12_emit_flush_xcs()
396 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen12_emit_flush_xcs()
399 if (gen12_needs_ccs_aux_inv(rq->engine) && in gen12_emit_flush_xcs()
400 rq->engine->class == COPY_ENGINE_CLASS) in gen12_emit_flush_xcs()
404 *cs++ = cmd; in gen12_emit_flush_xcs()
405 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen12_emit_flush_xcs()
406 *cs++ = 0; /* upper addr */ in gen12_emit_flush_xcs()
407 *cs++ = 0; /* value */ in gen12_emit_flush_xcs()
409 cs = gen12_emit_aux_table_inv(rq->engine, cs); in gen12_emit_flush_xcs()
412 *cs++ = preparser_disable(false); in gen12_emit_flush_xcs()
414 intel_ring_advance(rq, cs); in gen12_emit_flush_xcs()
416 return 0; in gen12_emit_flush_xcs()
421 return (i915_ggtt_offset(engine->status_page.vma) + in preempt_address()
430 tl = rcu_dereference_protected(rq->timeline, in hwsp_offset()
434 return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno); in hwsp_offset()
439 u32 *cs; in gen8_emit_init_breadcrumb() local
442 if (!i915_request_timeline(rq)->has_initial_breadcrumb) in gen8_emit_init_breadcrumb()
443 return 0; in gen8_emit_init_breadcrumb()
445 cs = intel_ring_begin(rq, 6); in gen8_emit_init_breadcrumb()
446 if (IS_ERR(cs)) in gen8_emit_init_breadcrumb()
447 return PTR_ERR(cs); in gen8_emit_init_breadcrumb()
449 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in gen8_emit_init_breadcrumb()
450 *cs++ = hwsp_offset(rq); in gen8_emit_init_breadcrumb()
451 *cs++ = 0; in gen8_emit_init_breadcrumb()
452 *cs++ = rq->fence.seqno - 1; in gen8_emit_init_breadcrumb()
462 * or spinning on a kernel semaphore (or earlier). For no-preemption in gen8_emit_init_breadcrumb()
471 *cs++ = MI_NOOP; in gen8_emit_init_breadcrumb()
472 *cs++ = MI_ARB_CHECK; in gen8_emit_init_breadcrumb()
474 intel_ring_advance(rq, cs); in gen8_emit_init_breadcrumb()
477 rq->infix = intel_ring_offset(rq, cs); in gen8_emit_init_breadcrumb()
479 __set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags); in gen8_emit_init_breadcrumb()
481 return 0; in gen8_emit_init_breadcrumb()
489 struct intel_context *ce = rq->context; in __xehp_emit_bb_start()
491 u32 *cs; in __xehp_emit_bb_start() local
493 GEM_BUG_ON(!ce->wa_bb_page); in __xehp_emit_bb_start()
495 cs = intel_ring_begin(rq, 12); in __xehp_emit_bb_start()
496 if (IS_ERR(cs)) in __xehp_emit_bb_start()
497 return PTR_ERR(cs); in __xehp_emit_bb_start()
499 *cs++ = MI_ARB_ON_OFF | arb; in __xehp_emit_bb_start()
501 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in __xehp_emit_bb_start()
504 *cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)); in __xehp_emit_bb_start()
505 *cs++ = wa_offset + DG2_PREDICATE_RESULT_WA; in __xehp_emit_bb_start()
506 *cs++ = 0; in __xehp_emit_bb_start()
508 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in __xehp_emit_bb_start()
509 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in __xehp_emit_bb_start()
510 *cs++ = lower_32_bits(offset); in __xehp_emit_bb_start()
511 *cs++ = upper_32_bits(offset); in __xehp_emit_bb_start()
514 *cs++ = MI_BATCH_BUFFER_START_GEN8; in __xehp_emit_bb_start()
515 *cs++ = wa_offset + DG2_PREDICATE_RESULT_BB; in __xehp_emit_bb_start()
516 *cs++ = 0; in __xehp_emit_bb_start()
518 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in __xehp_emit_bb_start()
520 intel_ring_advance(rq, cs); in __xehp_emit_bb_start()
522 return 0; in __xehp_emit_bb_start()
543 u32 *cs; in gen8_emit_bb_start_noarb() local
545 cs = intel_ring_begin(rq, 4); in gen8_emit_bb_start_noarb()
546 if (IS_ERR(cs)) in gen8_emit_bb_start_noarb()
547 return PTR_ERR(cs); in gen8_emit_bb_start_noarb()
559 * re-enabled before we close the request in gen8_emit_bb_start_noarb()
560 * (engine->emit_fini_breadcrumb). in gen8_emit_bb_start_noarb()
562 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start_noarb()
565 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start_noarb()
566 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in gen8_emit_bb_start_noarb()
567 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start_noarb()
568 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start_noarb()
570 intel_ring_advance(rq, cs); in gen8_emit_bb_start_noarb()
572 return 0; in gen8_emit_bb_start_noarb()
579 u32 *cs; in gen8_emit_bb_start() local
584 cs = intel_ring_begin(rq, 6); in gen8_emit_bb_start()
585 if (IS_ERR(cs)) in gen8_emit_bb_start()
586 return PTR_ERR(cs); in gen8_emit_bb_start()
588 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_bb_start()
590 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start()
591 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in gen8_emit_bb_start()
592 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start()
593 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start()
595 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start()
596 *cs++ = MI_NOOP; in gen8_emit_bb_start()
598 intel_ring_advance(rq, cs); in gen8_emit_bb_start()
600 return 0; in gen8_emit_bb_start()
605 struct intel_ring *ring __maybe_unused = rq->ring; in assert_request_valid()
608 GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0); in assert_request_valid()
616 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs) in gen8_emit_wa_tail() argument
618 /* Ensure there's always at least one preemption point per-request. */ in gen8_emit_wa_tail()
619 *cs++ = MI_ARB_CHECK; in gen8_emit_wa_tail()
620 *cs++ = MI_NOOP; in gen8_emit_wa_tail()
621 rq->wa_tail = intel_ring_offset(rq, cs); in gen8_emit_wa_tail()
626 return cs; in gen8_emit_wa_tail()
629 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs) in emit_preempt_busywait() argument
631 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in emit_preempt_busywait()
632 *cs++ = MI_SEMAPHORE_WAIT | in emit_preempt_busywait()
636 *cs++ = 0; in emit_preempt_busywait()
637 *cs++ = preempt_address(rq->engine); in emit_preempt_busywait()
638 *cs++ = 0; in emit_preempt_busywait()
639 *cs++ = MI_NOOP; in emit_preempt_busywait()
641 return cs; in emit_preempt_busywait()
645 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_tail() argument
647 *cs++ = MI_USER_INTERRUPT; in gen8_emit_fini_breadcrumb_tail()
649 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_fini_breadcrumb_tail()
650 if (intel_engine_has_semaphores(rq->engine) && in gen8_emit_fini_breadcrumb_tail()
651 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen8_emit_fini_breadcrumb_tail()
652 cs = emit_preempt_busywait(rq, cs); in gen8_emit_fini_breadcrumb_tail()
654 rq->tail = intel_ring_offset(rq, cs); in gen8_emit_fini_breadcrumb_tail()
655 assert_ring_tail_valid(rq->ring, rq->tail); in gen8_emit_fini_breadcrumb_tail()
657 return gen8_emit_wa_tail(rq, cs); in gen8_emit_fini_breadcrumb_tail()
660 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs) in emit_xcs_breadcrumb() argument
662 return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0); in emit_xcs_breadcrumb()
665 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_xcs() argument
667 return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); in gen8_emit_fini_breadcrumb_xcs()
670 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_rcs() argument
672 cs = gen8_emit_pipe_control(cs, in gen8_emit_fini_breadcrumb_rcs()
678 0); in gen8_emit_fini_breadcrumb_rcs()
681 cs = gen8_emit_ggtt_write_rcs(cs, in gen8_emit_fini_breadcrumb_rcs()
682 rq->fence.seqno, in gen8_emit_fini_breadcrumb_rcs()
687 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen8_emit_fini_breadcrumb_rcs()
690 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen11_emit_fini_breadcrumb_rcs() argument
692 cs = gen8_emit_pipe_control(cs, in gen11_emit_fini_breadcrumb_rcs()
699 0); in gen11_emit_fini_breadcrumb_rcs()
702 cs = gen8_emit_ggtt_write_rcs(cs, in gen11_emit_fini_breadcrumb_rcs()
703 rq->fence.seqno, in gen11_emit_fini_breadcrumb_rcs()
708 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen11_emit_fini_breadcrumb_rcs()
712 * Note that the CS instruction pre-parser will not stall on the breadcrumb
713 * flush and will continue pre-fetching the instructions after it before the
714 * memory sync is completed. On pre-gen12 HW, the pre-parser will stop at
715 * BB_START/END instructions, so, even though we might pre-fetch the pre-amble
718 * However, on gen12+ the parser can pre-fetch across the BB_START/END commands,
720 * the same intel_context, we might pre-fetch and then execute the pre-update
721 * instruction. To avoid this, the users of self-modifying code should either
724 * the in-kernel use-cases we've opted to use a separate context, see
726 * All the above applies only to the instructions themselves. Non-inline data
727 * used by the instructions is not pre-fetched.
730 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs) in gen12_emit_preempt_busywait() argument
732 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in gen12_emit_preempt_busywait()
733 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_preempt_busywait()
737 *cs++ = 0; in gen12_emit_preempt_busywait()
738 *cs++ = preempt_address(rq->engine); in gen12_emit_preempt_busywait()
739 *cs++ = 0; in gen12_emit_preempt_busywait()
740 *cs++ = 0; in gen12_emit_preempt_busywait()
742 return cs; in gen12_emit_preempt_busywait()
746 #define CCS_SEMAPHORE_PPHWSP_OFFSET 0x540
749 return i915_ggtt_offset(rq->context->state) + in ccs_semaphore_offset()
754 static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs) in ccs_emit_wa_busywait() argument
758 *cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL | in ccs_emit_wa_busywait()
760 *cs++ = ccs_semaphore_offset(rq); in ccs_emit_wa_busywait()
761 *cs++ = 0; in ccs_emit_wa_busywait()
762 *cs++ = 1; in ccs_emit_wa_busywait()
768 for (i = 0; i < 8; ++i) in ccs_emit_wa_busywait()
769 *cs++ = 0; in ccs_emit_wa_busywait()
771 *cs++ = MI_SEMAPHORE_WAIT | in ccs_emit_wa_busywait()
775 *cs++ = 0; in ccs_emit_wa_busywait()
776 *cs++ = ccs_semaphore_offset(rq); in ccs_emit_wa_busywait()
777 *cs++ = 0; in ccs_emit_wa_busywait()
779 return cs; in ccs_emit_wa_busywait()
783 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_tail() argument
785 *cs++ = MI_USER_INTERRUPT; in gen12_emit_fini_breadcrumb_tail()
787 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen12_emit_fini_breadcrumb_tail()
788 if (intel_engine_has_semaphores(rq->engine) && in gen12_emit_fini_breadcrumb_tail()
789 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen12_emit_fini_breadcrumb_tail()
790 cs = gen12_emit_preempt_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
793 if (intel_engine_uses_wa_hold_ccs_switchout(rq->engine)) in gen12_emit_fini_breadcrumb_tail()
794 cs = ccs_emit_wa_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
796 rq->tail = intel_ring_offset(rq, cs); in gen12_emit_fini_breadcrumb_tail()
797 assert_ring_tail_valid(rq->ring, rq->tail); in gen12_emit_fini_breadcrumb_tail()
799 return gen8_emit_wa_tail(rq, cs); in gen12_emit_fini_breadcrumb_tail()
802 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_xcs() argument
804 /* XXX Stalling flush before seqno write; post-sync not */ in gen12_emit_fini_breadcrumb_xcs()
805 cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0)); in gen12_emit_fini_breadcrumb_xcs()
806 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_xcs()
809 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_rcs() argument
811 struct drm_i915_private *i915 = rq->i915; in gen12_emit_fini_breadcrumb_rcs()
812 struct intel_gt *gt = rq->engine->gt; in gen12_emit_fini_breadcrumb_rcs()
821 if (GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70)) in gen12_emit_fini_breadcrumb_rcs()
827 cs = gen12_emit_pipe_control(cs, 0, in gen12_emit_fini_breadcrumb_rcs()
828 PIPE_CONTROL_DEPTH_CACHE_FLUSH, 0); in gen12_emit_fini_breadcrumb_rcs()
834 if (!HAS_3D_PIPELINE(rq->i915)) in gen12_emit_fini_breadcrumb_rcs()
836 else if (rq->engine->class == COMPUTE_CLASS) in gen12_emit_fini_breadcrumb_rcs()
839 cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0); in gen12_emit_fini_breadcrumb_rcs()
842 cs = gen12_emit_ggtt_write_rcs(cs, in gen12_emit_fini_breadcrumb_rcs()
843 rq->fence.seqno, in gen12_emit_fini_breadcrumb_rcs()
845 0, in gen12_emit_fini_breadcrumb_rcs()
849 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_rcs()