25  * pipeline (i.e., CCS engines).
52 			      u32 *dw, int i)  in emit_aux_table_inv()  argument
54 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;  in emit_aux_table_inv()
55 	dw[i++] = reg.addr + gt->mmio.adj_offset;  in emit_aux_table_inv()
56 	dw[i++] = AUX_INV;  in emit_aux_table_inv()
57 	dw[i++] = MI_NOOP;  in emit_aux_table_inv()
59 	return i;  in emit_aux_table_inv()
62 static int emit_user_interrupt(u32 *dw, int i)  in emit_user_interrupt()  argument
64 	dw[i++] = MI_USER_INTERRUPT;  in emit_user_interrupt()
65 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in emit_user_interrupt()
66 	dw[i++] = MI_ARB_CHECK;  in emit_user_interrupt()
68 	return i;  in emit_user_interrupt()
71 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)  in emit_store_imm_ggtt()  argument
73 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);  in emit_store_imm_ggtt()
74 	dw[i++] = addr;  in emit_store_imm_ggtt()
75 	dw[i++] = 0;  in emit_store_imm_ggtt()
76 	dw[i++] = value;  in emit_store_imm_ggtt()
78 	return i;  in emit_store_imm_ggtt()
82 			       u32 *dw, int i)  in emit_flush_imm_ggtt()  argument
84 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_imm_ggtt()
86 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;  in emit_flush_imm_ggtt()
87 	dw[i++] = 0;  in emit_flush_imm_ggtt()
88 	dw[i++] = value;  in emit_flush_imm_ggtt()
90 	return i;  in emit_flush_imm_ggtt()
93 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)  in emit_bb_start()  argument
95 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);  in emit_bb_start()
96 	dw[i++] = lower_32_bits(batch_addr);  in emit_bb_start()
97 	dw[i++] = upper_32_bits(batch_addr);  in emit_bb_start()
99 	return i;  in emit_bb_start()
102 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)  in emit_flush_invalidate()  argument
104 	dw[i] = MI_FLUSH_DW;  in emit_flush_invalidate()
105 	dw[i] |= flag;  in emit_flush_invalidate()
106 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_invalidate()
109 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;  in emit_flush_invalidate()
110 	dw[i++] = 0;  in emit_flush_invalidate()
111 	dw[i++] = ~0U;  in emit_flush_invalidate()
113 	return i;  in emit_flush_invalidate()
117 				int i)  in emit_pipe_invalidate()  argument
134 	dw[i++] = GFX_OP_PIPE_CONTROL(6);  in emit_pipe_invalidate()
135 	dw[i++] = flags;  in emit_pipe_invalidate()
136 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR;  in emit_pipe_invalidate()
137 	dw[i++] = 0;  in emit_pipe_invalidate()
138 	dw[i++] = 0;  in emit_pipe_invalidate()
139 	dw[i++] = 0;  in emit_pipe_invalidate()
141 	return i;  in emit_pipe_invalidate()
145 				       u32 *dw, int i)  in emit_store_imm_ppgtt_posted()  argument
147 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);  in emit_store_imm_ppgtt_posted()
148 	dw[i++] = lower_32_bits(addr);  in emit_store_imm_ppgtt_posted()
149 	dw[i++] = upper_32_bits(addr);  in emit_store_imm_ppgtt_posted()
150 	dw[i++] = lower_32_bits(value);  in emit_store_imm_ppgtt_posted()
151 	dw[i++] = upper_32_bits(value);  in emit_store_imm_ppgtt_posted()
153 	return i;  in emit_store_imm_ppgtt_posted()
156 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)  in emit_render_cache_flush()  argument
177 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | PIPE_CONTROL0_HDC_PIPELINE_FLUSH;  in emit_render_cache_flush()
178 	dw[i++] = flags;  in emit_render_cache_flush()
179 	dw[i++] = 0;  in emit_render_cache_flush()
180 	dw[i++] = 0;  in emit_render_cache_flush()
181 	dw[i++] = 0;  in emit_render_cache_flush()
182 	dw[i++] = 0;  in emit_render_cache_flush()
184 	return i;  in emit_render_cache_flush()
187 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)  in emit_pipe_control_to_ring_end()  argument
190 		return i;  in emit_pipe_control_to_ring_end()
193 		dw[i++] = GFX_OP_PIPE_CONTROL(6);  in emit_pipe_control_to_ring_end()
194 		dw[i++] = PIPE_CONTROL_LRI_POST_SYNC;  in emit_pipe_control_to_ring_end()
195 		dw[i++] = RING_NOPID(hwe->mmio_base).addr;  in emit_pipe_control_to_ring_end()
196 		dw[i++] = 0;  in emit_pipe_control_to_ring_end()
197 		dw[i++] = 0;  in emit_pipe_control_to_ring_end()
198 		dw[i++] = 0;  in emit_pipe_control_to_ring_end()
201 	return i;  in emit_pipe_control_to_ring_end()
205 			      int i)  in emit_pipe_imm_ggtt()  argument
207 	dw[i++] = GFX_OP_PIPE_CONTROL(6);  in emit_pipe_imm_ggtt()
208 	dw[i++] = (stall_only ? PIPE_CONTROL_CS_STALL :  in emit_pipe_imm_ggtt()
211 	dw[i++] = addr;  in emit_pipe_imm_ggtt()
212 	dw[i++] = 0;  in emit_pipe_imm_ggtt()
213 	dw[i++] = value;  in emit_pipe_imm_ggtt()
214 	dw[i++] = 0; /* We're thrashing one extra dword. */  in emit_pipe_imm_ggtt()
216 	return i;  in emit_pipe_imm_ggtt()
228 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_simple()  local
234 		dw[i++] = preparser_disable(true);  in __emit_job_gen12_simple()
235 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_simple()
236 					seqno, true, dw, i);  in __emit_job_gen12_simple()
237 		dw[i++] = preparser_disable(false);  in __emit_job_gen12_simple()
239 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_simple()
240 					seqno, dw, i);  in __emit_job_gen12_simple()
243 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_simple()
246 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_simple()
248 						dw, i);  in __emit_job_gen12_simple()
250 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_simple()
252 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_simple()
254 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_simple()
256 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_simple()
276 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_video()  local
283 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_video()
288 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);  in __emit_job_gen12_video()
290 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);  in __emit_job_gen12_video()
294 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_video()
295 					seqno, true, dw, i);  in __emit_job_gen12_video()
297 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_video()
300 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_video()
301 					seqno, dw, i);  in __emit_job_gen12_video()
303 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_video()
306 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_video()
308 						dw, i);  in __emit_job_gen12_video()
310 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_video()
312 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_video()
314 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_video()
316 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_video()
323 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_render_compute()  local
331 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_render_compute()
338 	i = emit_pipe_invalidate(mask_flags, vm && vm->batch_invalidate_tlb, dw, i);  in __emit_job_gen12_render_compute()
342 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);  in __emit_job_gen12_render_compute()
344 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_render_compute()
346 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_render_compute()
347 				seqno, dw, i);  in __emit_job_gen12_render_compute()
349 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_render_compute()
351 	i = emit_render_cache_flush(job, dw, i);  in __emit_job_gen12_render_compute()
354 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_render_compute()
356 						dw, i);  in __emit_job_gen12_render_compute()
358 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);  in __emit_job_gen12_render_compute()
360 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_render_compute()
362 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);  in __emit_job_gen12_render_compute()
364 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_render_compute()
366 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_render_compute()
372 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in emit_migration_job_gen12()  local
374 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in emit_migration_job_gen12()
375 				seqno, dw, i);  in emit_migration_job_gen12()
377 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */  in emit_migration_job_gen12()
379 	i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i);  in emit_migration_job_gen12()
382 	dw[i++] = preparser_disable(true);  in emit_migration_job_gen12()
383 	i = emit_flush_invalidate(0, dw, i);  in emit_migration_job_gen12()
384 	dw[i++] = preparser_disable(false);  in emit_migration_job_gen12()
386 	i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i);  in emit_migration_job_gen12()
388 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |  in emit_migration_job_gen12()
390 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;  in emit_migration_job_gen12()
391 	dw[i++] = 0;  in emit_migration_job_gen12()
392 	dw[i++] = seqno; /* value */  in emit_migration_job_gen12()
394 	i = emit_user_interrupt(dw, i);  in emit_migration_job_gen12()
396 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);  in emit_migration_job_gen12()
398 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in emit_migration_job_gen12()
414 	int i;  in emit_job_gen12_copy()  local
422 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_copy()
423 		__emit_job_gen12_simple(job, job->q->lrc + i,  in emit_job_gen12_copy()
424 				        job->batch_addr[i],  in emit_job_gen12_copy()
430 	int i;  in emit_job_gen12_video()  local
433 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_video()
434 		__emit_job_gen12_video(job, job->q->lrc + i,  in emit_job_gen12_video()
435 				       job->batch_addr[i],  in emit_job_gen12_video()
441 	int i;  in emit_job_gen12_render_compute()  local
443 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_render_compute()
444 		__emit_job_gen12_render_compute(job, job->q->lrc + i,  in emit_job_gen12_render_compute()
445 						job->batch_addr[i],  in emit_job_gen12_render_compute()