Lines Matching full:i
27 * pipeline (i.e., CCS engines).
54 u32 *dw, int i)
56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 dw[i++] = reg.addr + gt->mmio.adj_offset;
58 dw[i++] = AUX_INV;
59 dw[i++] = MI_NOOP;
61 return i;
64 static int emit_user_interrupt(u32 *dw, int i)
66 dw[i++] = MI_USER_INTERRUPT;
67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 dw[i++] = MI_ARB_CHECK;
70 return i;
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 dw[i++] = addr;
77 dw[i++] = 0;
78 dw[i++] = value;
80 return i;
83 static int emit_flush_dw(u32 *dw, int i)
85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 dw[i++] = 0;
87 dw[i++] = 0;
88 dw[i++] = 0;
90 return i;
93 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
95 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
97 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
98 dw[i++] = 0;
99 dw[i++] = value;
101 return i;
104 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
106 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
107 dw[i++] = lower_32_bits(batch_addr);
108 dw[i++] = upper_32_bits(batch_addr);
110 return i;
113 static int emit_flush_invalidate(u32 addr, u32 val, u32 *dw, int i)
115 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW |
118 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
119 dw[i++] = 0;
120 dw[i++] = val;
122 return i;
126 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
128 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
129 dw[i++] = bit_group_1;
130 dw[i++] = offset;
131 dw[i++] = 0;
132 dw[i++] = value;
133 dw[i++] = 0;
135 return i;
139 int i)
160 return emit_pipe_control(dw, i, flags0, flags1,
165 u32 *dw, int i)
167 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
168 dw[i++] = lower_32_bits(addr);
169 dw[i++] = upper_32_bits(addr);
170 dw[i++] = lower_32_bits(value);
171 dw[i++] = upper_32_bits(value);
173 return i;
176 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
183 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
201 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
204 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
207 return i;
210 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
213 return i;
217 int i)
225 return emit_pipe_control(dw, i, 0, flags, addr, value);
236 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
238 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
239 dw[i++] = RING_CTX_TIMESTAMP(0).addr;
240 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
241 dw[i++] = 0;
243 return i;
250 u32 dw[MAX_JOB_SIZE_DW], i = 0;
254 i = emit_copy_timestamp(lrc, dw, i);
257 dw[i++] = preparser_disable(true);
258 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
259 seqno, MI_INVALIDATE_TLB, dw, i);
260 dw[i++] = preparser_disable(false);
262 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
263 seqno, dw, i);
266 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
269 i = emit_flush_dw(dw, i);
270 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
272 dw, i);
275 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
277 i = emit_user_interrupt(dw, i);
279 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
281 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
301 u32 dw[MAX_JOB_SIZE_DW], i = 0;
307 i = emit_copy_timestamp(lrc, dw, i);
309 dw[i++] = preparser_disable(true);
314 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
316 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
320 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
321 seqno, MI_INVALIDATE_TLB, dw, i);
323 dw[i++] = preparser_disable(false);
326 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
327 seqno, dw, i);
329 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
332 i = emit_flush_dw(dw, i);
333 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
335 dw, i);
338 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
340 i = emit_user_interrupt(dw, i);
342 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
344 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
351 u32 dw[MAX_JOB_SIZE_DW], i = 0;
358 i = emit_copy_timestamp(lrc, dw, i);
360 dw[i++] = preparser_disable(true);
367 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
371 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
373 dw[i++] = preparser_disable(false);
375 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
376 seqno, dw, i);
378 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
380 i = emit_render_cache_flush(job, dw, i);
383 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
385 dw, i);
387 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
389 i = emit_user_interrupt(dw, i);
391 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
393 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
395 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
402 u32 dw[MAX_JOB_SIZE_DW], i = 0;
404 i = emit_copy_timestamp(lrc, dw, i);
406 i = emit_store_imm_ggtt(saddr, seqno, dw, i);
408 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
410 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
412 dw[i++] = preparser_disable(true);
413 i = emit_flush_invalidate(saddr, seqno, dw, i);
414 dw[i++] = preparser_disable(false);
416 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
418 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
420 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
421 dw[i++] = 0;
422 dw[i++] = seqno; /* value */
424 i = emit_user_interrupt(dw, i);
426 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
428 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
444 int i;
452 for (i = 0; i < job->q->width; ++i)
453 __emit_job_gen12_simple(job, job->q->lrc[i],
454 job->ptrs[i].batch_addr,
460 int i;
463 for (i = 0; i < job->q->width; ++i)
464 __emit_job_gen12_video(job, job->q->lrc[i],
465 job->ptrs[i].batch_addr,
471 int i;
473 for (i = 0; i < job->q->width; ++i)
474 __emit_job_gen12_render_compute(job, job->q->lrc[i],
475 job->ptrs[i].batch_addr,