54 			      u32 *dw, int i)  in emit_aux_table_inv()  argument
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;  in emit_aux_table_inv()
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;  in emit_aux_table_inv()
58 	dw[i++] = AUX_INV;  in emit_aux_table_inv()
59 	dw[i++] = MI_NOOP;  in emit_aux_table_inv()
64 static int emit_user_interrupt(u32 *dw, int i)  in emit_user_interrupt()  argument
66 	dw[i++] = MI_USER_INTERRUPT;  in emit_user_interrupt()
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in emit_user_interrupt()
68 	dw[i++] = MI_ARB_CHECK;  in emit_user_interrupt()
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)  in emit_store_imm_ggtt()  argument
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);  in emit_store_imm_ggtt()
76 	dw[i++] = addr;  in emit_store_imm_ggtt()
77 	dw[i++] = 0;  in emit_store_imm_ggtt()
78 	dw[i++] = value;  in emit_store_imm_ggtt()
83 static int emit_flush_dw(u32 *dw, int i)  in emit_flush_dw()  argument
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;  in emit_flush_dw()
86 	dw[i++] = 0;  in emit_flush_dw()
87 	dw[i++] = 0;  in emit_flush_dw()
88 	dw[i++] = 0;  in emit_flush_dw()
93 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)  in emit_flush_imm_ggtt()  argument
95 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_imm_ggtt()
97 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;  in emit_flush_imm_ggtt()
98 	dw[i++] = 0;  in emit_flush_imm_ggtt()
99 	dw[i++] = value;  in emit_flush_imm_ggtt()
104 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)  in emit_bb_start()  argument
106 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);  in emit_bb_start()
107 	dw[i++] = lower_32_bits(batch_addr);  in emit_bb_start()
108 	dw[i++] = upper_32_bits(batch_addr);  in emit_bb_start()
113 static int emit_flush_invalidate(u32 *dw, int i)  in emit_flush_invalidate()  argument
115 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW |  in emit_flush_invalidate()
117 	dw[i++] = LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR;  in emit_flush_invalidate()
118 	dw[i++] = 0;  in emit_flush_invalidate()
119 	dw[i++] = 0;  in emit_flush_invalidate()
125 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)  in emit_pipe_control()  argument
127 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;  in emit_pipe_control()
128 	dw[i++] = bit_group_1;  in emit_pipe_control()
129 	dw[i++] = offset;  in emit_pipe_control()
130 	dw[i++] = 0;  in emit_pipe_control()
131 	dw[i++] = value;  in emit_pipe_control()
132 	dw[i++] = 0;  in emit_pipe_control()
137 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,  in emit_pipe_invalidate()  argument
159 	return emit_pipe_control(dw, i, flags0, flags1,  in emit_pipe_invalidate()
164 				       u32 *dw, int i)  in emit_store_imm_ppgtt_posted()  argument
166 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);  in emit_store_imm_ppgtt_posted()
167 	dw[i++] = lower_32_bits(addr);  in emit_store_imm_ppgtt_posted()
168 	dw[i++] = upper_32_bits(addr);  in emit_store_imm_ppgtt_posted()
169 	dw[i++] = lower_32_bits(value);  in emit_store_imm_ppgtt_posted()
170 	dw[i++] = upper_32_bits(value);  in emit_store_imm_ppgtt_posted()
175 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)  in emit_render_cache_flush()  argument
182 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,  in emit_render_cache_flush()
200 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);  in emit_render_cache_flush()
203 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)  in emit_pipe_control_to_ring_end()  argument
209 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,  in emit_pipe_control_to_ring_end()
215 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,  in emit_pipe_imm_ggtt()  argument
224 	return emit_pipe_control(dw, i, 0, flags, addr, value);  in emit_pipe_imm_ggtt()
235 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)  in emit_copy_timestamp()  argument
237 	dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;  in emit_copy_timestamp()
238 	dw[i++] = RING_CTX_TIMESTAMP(0).addr;  in emit_copy_timestamp()
239 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);  in emit_copy_timestamp()
240 	dw[i++] = 0;  in emit_copy_timestamp()
249 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_simple()  local
253 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_simple()
256 		dw[i++] = preparser_disable(true);  in __emit_job_gen12_simple()
258 					seqno, MI_INVALIDATE_TLB, dw, i);  in __emit_job_gen12_simple()
259 		dw[i++] = preparser_disable(false);  in __emit_job_gen12_simple()
262 					seqno, dw, i);  in __emit_job_gen12_simple()
265 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_simple()
268 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_simple()
271 						dw, i);  in __emit_job_gen12_simple()
274 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);  in __emit_job_gen12_simple()
276 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_simple()
280 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_simple()
300 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_video()  local
306 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_video()
308 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_video()
313 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);  in __emit_job_gen12_video()
315 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);  in __emit_job_gen12_video()
320 					seqno, MI_INVALIDATE_TLB, dw, i);  in __emit_job_gen12_video()
322 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_video()
326 					seqno, dw, i);  in __emit_job_gen12_video()
328 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_video()
331 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_video()
334 						dw, i);  in __emit_job_gen12_video()
337 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);  in __emit_job_gen12_video()
339 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_video()
343 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_video()
350 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_render_compute()  local
357 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_render_compute()
359 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_render_compute()
366 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);  in __emit_job_gen12_render_compute()
370 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);  in __emit_job_gen12_render_compute()
372 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_render_compute()
375 				seqno, dw, i);  in __emit_job_gen12_render_compute()
377 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_render_compute()
379 	i = emit_render_cache_flush(job, dw, i);  in __emit_job_gen12_render_compute()
384 						dw, i);  in __emit_job_gen12_render_compute()
386 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);  in __emit_job_gen12_render_compute()
388 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_render_compute()
390 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);  in __emit_job_gen12_render_compute()
394 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_render_compute()
400 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in emit_migration_job_gen12()  local
402 	i = emit_copy_timestamp(lrc, dw, i);  in emit_migration_job_gen12()
405 				seqno, dw, i);  in emit_migration_job_gen12()
407 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */  in emit_migration_job_gen12()
409 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
413 		dw[i++] = preparser_disable(true);  in emit_migration_job_gen12()
414 		i = emit_flush_invalidate(dw, i);  in emit_migration_job_gen12()
415 		dw[i++] = preparser_disable(false);  in emit_migration_job_gen12()
418 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
420 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |  in emit_migration_job_gen12()
422 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;  in emit_migration_job_gen12()
423 	dw[i++] = 0;  in emit_migration_job_gen12()
424 	dw[i++] = seqno; /* value */  in emit_migration_job_gen12()
426 	i = emit_user_interrupt(dw, i);  in emit_migration_job_gen12()
430 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in emit_migration_job_gen12()