16 	u32 *cs, flags = 0;  in gen8_emit_flush_rcs()  local
58 	cs = intel_ring_begin(rq, len);  in gen8_emit_flush_rcs()
59 	if (IS_ERR(cs))  in gen8_emit_flush_rcs()
60 		return PTR_ERR(cs);  in gen8_emit_flush_rcs()
63 		cs = gen8_emit_pipe_control(cs, 0, 0);  in gen8_emit_flush_rcs()
66 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,  in gen8_emit_flush_rcs()
69 	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen8_emit_flush_rcs()
72 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);  in gen8_emit_flush_rcs()
74 	intel_ring_advance(rq, cs);  in gen8_emit_flush_rcs()
81 	u32 cmd, *cs;  in gen8_emit_flush_xcs()  local
83 	cs = intel_ring_begin(rq, 4);  in gen8_emit_flush_xcs()
84 	if (IS_ERR(cs))  in gen8_emit_flush_xcs()
85 		return PTR_ERR(cs);  in gen8_emit_flush_xcs()
103 	*cs++ = cmd;  in gen8_emit_flush_xcs()
104 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen8_emit_flush_xcs()
105 	*cs++ = 0; /* upper addr */  in gen8_emit_flush_xcs()
106 	*cs++ = 0; /* value */  in gen8_emit_flush_xcs()
107 	intel_ring_advance(rq, cs);  in gen8_emit_flush_xcs()
115 		u32 *cs;  in gen11_emit_flush_rcs()  local
128 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
129 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
130 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
132 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
133 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
137 		u32 *cs;  in gen11_emit_flush_rcs()  local
152 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
153 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
154 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
156 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
157 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
202 u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)  in gen12_emit_aux_table_inv()  argument
208 		return cs;  in gen12_emit_aux_table_inv()
210 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;  in gen12_emit_aux_table_inv()
211 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
212 	*cs++ = AUX_INV;  in gen12_emit_aux_table_inv()
214 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_aux_table_inv()
218 	*cs++ = 0;  in gen12_emit_aux_table_inv()
219 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
220 	*cs++ = 0;  in gen12_emit_aux_table_inv()
221 	*cs++ = 0;  in gen12_emit_aux_table_inv()
223 	return cs;  in gen12_emit_aux_table_inv()
231 		u32 *cs;  in mtl_dummy_pipe_control()  local
234 		cs = intel_ring_begin(rq, 6);  in mtl_dummy_pipe_control()
235 		if (IS_ERR(cs))  in mtl_dummy_pipe_control()
236 			return PTR_ERR(cs);  in mtl_dummy_pipe_control()
237 		cs = gen12_emit_pipe_control(cs,  in mtl_dummy_pipe_control()
241 		intel_ring_advance(rq, cs);  in mtl_dummy_pipe_control()
259 		u32 *cs;  in gen12_emit_flush_rcs()  local
303 		cs = intel_ring_begin(rq, 6);  in gen12_emit_flush_rcs()
304 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
305 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
307 		cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1,  in gen12_emit_flush_rcs()
309 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
314 		u32 *cs, count;  in gen12_emit_flush_rcs()  local
343 		cs = intel_ring_begin(rq, count);  in gen12_emit_flush_rcs()
344 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
345 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
352 		*cs++ = preparser_disable(true);  in gen12_emit_flush_rcs()
354 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen12_emit_flush_rcs()
356 		cs = gen12_emit_aux_table_inv(engine, cs);  in gen12_emit_flush_rcs()
358 		*cs++ = preparser_disable(false);  in gen12_emit_flush_rcs()
359 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
368 	u32 *cs;  in gen12_emit_flush_xcs()  local
377 	cs = intel_ring_begin(rq, cmd);  in gen12_emit_flush_xcs()
378 	if (IS_ERR(cs))  in gen12_emit_flush_xcs()
379 		return PTR_ERR(cs);  in gen12_emit_flush_xcs()
382 		*cs++ = preparser_disable(true);  in gen12_emit_flush_xcs()
404 	*cs++ = cmd;  in gen12_emit_flush_xcs()
405 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen12_emit_flush_xcs()
406 	*cs++ = 0; /* upper addr */  in gen12_emit_flush_xcs()
407 	*cs++ = 0; /* value */  in gen12_emit_flush_xcs()
409 	cs = gen12_emit_aux_table_inv(rq->engine, cs);  in gen12_emit_flush_xcs()
412 		*cs++ = preparser_disable(false);  in gen12_emit_flush_xcs()
414 	intel_ring_advance(rq, cs);  in gen12_emit_flush_xcs()
439 	u32 *cs;  in gen8_emit_init_breadcrumb()  local
445 	cs = intel_ring_begin(rq, 6);  in gen8_emit_init_breadcrumb()
446 	if (IS_ERR(cs))  in gen8_emit_init_breadcrumb()
447 		return PTR_ERR(cs);  in gen8_emit_init_breadcrumb()
449 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in gen8_emit_init_breadcrumb()
450 	*cs++ = hwsp_offset(rq);  in gen8_emit_init_breadcrumb()
451 	*cs++ = 0;  in gen8_emit_init_breadcrumb()
452 	*cs++ = rq->fence.seqno - 1;  in gen8_emit_init_breadcrumb()
471 	*cs++ = MI_NOOP;  in gen8_emit_init_breadcrumb()
472 	*cs++ = MI_ARB_CHECK;  in gen8_emit_init_breadcrumb()
474 	intel_ring_advance(rq, cs);  in gen8_emit_init_breadcrumb()
477 	rq->infix = intel_ring_offset(rq, cs);  in gen8_emit_init_breadcrumb()
491 	u32 *cs;  in __xehp_emit_bb_start()  local
495 	cs = intel_ring_begin(rq, 12);  in __xehp_emit_bb_start()
496 	if (IS_ERR(cs))  in __xehp_emit_bb_start()
497 		return PTR_ERR(cs);  in __xehp_emit_bb_start()
499 	*cs++ = MI_ARB_ON_OFF | arb;  in __xehp_emit_bb_start()
501 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in __xehp_emit_bb_start()
504 	*cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0));  in __xehp_emit_bb_start()
505 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_WA;  in __xehp_emit_bb_start()
506 	*cs++ = 0;  in __xehp_emit_bb_start()
508 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in __xehp_emit_bb_start()
510 	*cs++ = lower_32_bits(offset);  in __xehp_emit_bb_start()
511 	*cs++ = upper_32_bits(offset);  in __xehp_emit_bb_start()
514 	*cs++ = MI_BATCH_BUFFER_START_GEN8;  in __xehp_emit_bb_start()
515 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_BB;  in __xehp_emit_bb_start()
516 	*cs++ = 0;  in __xehp_emit_bb_start()
518 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in __xehp_emit_bb_start()
520 	intel_ring_advance(rq, cs);  in __xehp_emit_bb_start()
543 	u32 *cs;  in gen8_emit_bb_start_noarb()  local
545 	cs = intel_ring_begin(rq, 4);  in gen8_emit_bb_start_noarb()
546 	if (IS_ERR(cs))  in gen8_emit_bb_start_noarb()
547 		return PTR_ERR(cs);  in gen8_emit_bb_start_noarb()
562 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start_noarb()
565 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start_noarb()
567 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start_noarb()
568 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start_noarb()
570 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start_noarb()
579 	u32 *cs;  in gen8_emit_bb_start()  local
584 	cs = intel_ring_begin(rq, 6);  in gen8_emit_bb_start()
585 	if (IS_ERR(cs))  in gen8_emit_bb_start()
586 		return PTR_ERR(cs);  in gen8_emit_bb_start()
588 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_bb_start()
590 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start()
592 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start()
593 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start()
595 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start()
596 	*cs++ = MI_NOOP;  in gen8_emit_bb_start()
598 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start()
616 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_wa_tail()  argument
619 	*cs++ = MI_ARB_CHECK;  in gen8_emit_wa_tail()
620 	*cs++ = MI_NOOP;  in gen8_emit_wa_tail()
621 	rq->wa_tail = intel_ring_offset(rq, cs);  in gen8_emit_wa_tail()
626 	return cs;  in gen8_emit_wa_tail()
629 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in emit_preempt_busywait()  argument
631 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in emit_preempt_busywait()
632 	*cs++ = MI_SEMAPHORE_WAIT |  in emit_preempt_busywait()
636 	*cs++ = 0;  in emit_preempt_busywait()
637 	*cs++ = preempt_address(rq->engine);  in emit_preempt_busywait()
638 	*cs++ = 0;  in emit_preempt_busywait()
639 	*cs++ = MI_NOOP;  in emit_preempt_busywait()
641 	return cs;  in emit_preempt_busywait()
645 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_tail()  argument
647 	*cs++ = MI_USER_INTERRUPT;  in gen8_emit_fini_breadcrumb_tail()
649 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_fini_breadcrumb_tail()
652 		cs = emit_preempt_busywait(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
654 	rq->tail = intel_ring_offset(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
657 	return gen8_emit_wa_tail(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
660 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)  in emit_xcs_breadcrumb()  argument
662 	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);  in emit_xcs_breadcrumb()
665 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_xcs()  argument
667 	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));  in gen8_emit_fini_breadcrumb_xcs()
670 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_rcs()  argument
672 	cs = gen8_emit_pipe_control(cs,  in gen8_emit_fini_breadcrumb_rcs()
681 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen8_emit_fini_breadcrumb_rcs()
687 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen8_emit_fini_breadcrumb_rcs()
690 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen11_emit_fini_breadcrumb_rcs()  argument
692 	cs = gen8_emit_pipe_control(cs,  in gen11_emit_fini_breadcrumb_rcs()
702 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen11_emit_fini_breadcrumb_rcs()
708 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen11_emit_fini_breadcrumb_rcs()
712  * Note that the CS instruction pre-parser will not stall on the breadcrumb
730 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in gen12_emit_preempt_busywait()  argument
732 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in gen12_emit_preempt_busywait()
733 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_preempt_busywait()
737 	*cs++ = 0;  in gen12_emit_preempt_busywait()
738 	*cs++ = preempt_address(rq->engine);  in gen12_emit_preempt_busywait()
739 	*cs++ = 0;  in gen12_emit_preempt_busywait()
740 	*cs++ = 0;  in gen12_emit_preempt_busywait()
742 	return cs;  in gen12_emit_preempt_busywait()
754 static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs)  in ccs_emit_wa_busywait()  argument
758 	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |  in ccs_emit_wa_busywait()
760 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
761 	*cs++ = 0;  in ccs_emit_wa_busywait()
762 	*cs++ = 1;  in ccs_emit_wa_busywait()
769 		*cs++ = 0;  in ccs_emit_wa_busywait()
771 	*cs++ = MI_SEMAPHORE_WAIT |  in ccs_emit_wa_busywait()
775 	*cs++ = 0;  in ccs_emit_wa_busywait()
776 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
777 	*cs++ = 0;  in ccs_emit_wa_busywait()
779 	return cs;  in ccs_emit_wa_busywait()
783 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_tail()  argument
785 	*cs++ = MI_USER_INTERRUPT;  in gen12_emit_fini_breadcrumb_tail()
787 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen12_emit_fini_breadcrumb_tail()
790 		cs = gen12_emit_preempt_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
794 		cs = ccs_emit_wa_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
796 	rq->tail = intel_ring_offset(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
799 	return gen8_emit_wa_tail(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
802 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_xcs()  argument
805 	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));  in gen12_emit_fini_breadcrumb_xcs()
806 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_xcs()
809 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_rcs()  argument
827 		cs = gen12_emit_pipe_control(cs, 0,  in gen12_emit_fini_breadcrumb_rcs()
839 	cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0);  in gen12_emit_fini_breadcrumb_rcs()
842 	cs = gen12_emit_ggtt_write_rcs(cs,  in gen12_emit_fini_breadcrumb_rcs()
849 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_rcs()