Lines Matching +full:cs +full:- +full:3
1 // SPDX-License-Identifier: MIT
17 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
21 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
22 * produced by non-pipelined state commands), software needs to first
23 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
26 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
27 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
31 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
32 * BEFORE the pipe-control with a post-sync op and no write-cache
36 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
40 * - Render Target Cache Flush Enable ([12] of DW1)
41 * - Depth Cache Flush Enable ([0] of DW1)
42 * - Stall at Pixel Scoreboard ([1] of DW1)
43 * - Depth Stall ([13] of DW1)
44 * - Post-Sync Operation ([13] of DW1)
45 * - Notify Enable ([8] of DW1)"
49 * Post-sync nonzero is what triggered this second workaround, so we
57 intel_gt_scratch_offset(rq->engine->gt, in gen6_emit_post_sync_nonzero_flush()
59 u32 *cs; in gen6_emit_post_sync_nonzero_flush() local
61 cs = intel_ring_begin(rq, 6); in gen6_emit_post_sync_nonzero_flush()
62 if (IS_ERR(cs)) in gen6_emit_post_sync_nonzero_flush()
63 return PTR_ERR(cs); in gen6_emit_post_sync_nonzero_flush()
65 *cs++ = GFX_OP_PIPE_CONTROL(5); in gen6_emit_post_sync_nonzero_flush()
66 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; in gen6_emit_post_sync_nonzero_flush()
67 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; in gen6_emit_post_sync_nonzero_flush()
68 *cs++ = 0; /* low dword */ in gen6_emit_post_sync_nonzero_flush()
69 *cs++ = 0; /* high dword */ in gen6_emit_post_sync_nonzero_flush()
70 *cs++ = MI_NOOP; in gen6_emit_post_sync_nonzero_flush()
71 intel_ring_advance(rq, cs); in gen6_emit_post_sync_nonzero_flush()
73 cs = intel_ring_begin(rq, 6); in gen6_emit_post_sync_nonzero_flush()
74 if (IS_ERR(cs)) in gen6_emit_post_sync_nonzero_flush()
75 return PTR_ERR(cs); in gen6_emit_post_sync_nonzero_flush()
77 *cs++ = GFX_OP_PIPE_CONTROL(5); in gen6_emit_post_sync_nonzero_flush()
78 *cs++ = PIPE_CONTROL_QW_WRITE; in gen6_emit_post_sync_nonzero_flush()
79 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; in gen6_emit_post_sync_nonzero_flush()
80 *cs++ = 0; in gen6_emit_post_sync_nonzero_flush()
81 *cs++ = 0; in gen6_emit_post_sync_nonzero_flush()
82 *cs++ = MI_NOOP; in gen6_emit_post_sync_nonzero_flush()
83 intel_ring_advance(rq, cs); in gen6_emit_post_sync_nonzero_flush()
91 intel_gt_scratch_offset(rq->engine->gt, in gen6_emit_flush_rcs()
93 u32 *cs, flags = 0; in gen6_emit_flush_rcs() local
124 * TLB invalidate requires a post-sync write. in gen6_emit_flush_rcs()
129 cs = intel_ring_begin(rq, 4); in gen6_emit_flush_rcs()
130 if (IS_ERR(cs)) in gen6_emit_flush_rcs()
131 return PTR_ERR(cs); in gen6_emit_flush_rcs()
133 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen6_emit_flush_rcs()
134 *cs++ = flags; in gen6_emit_flush_rcs()
135 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; in gen6_emit_flush_rcs()
136 *cs++ = 0; in gen6_emit_flush_rcs()
137 intel_ring_advance(rq, cs); in gen6_emit_flush_rcs()
142 u32 *gen6_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen6_emit_breadcrumb_rcs() argument
145 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen6_emit_breadcrumb_rcs()
146 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; in gen6_emit_breadcrumb_rcs()
147 *cs++ = 0; in gen6_emit_breadcrumb_rcs()
148 *cs++ = 0; in gen6_emit_breadcrumb_rcs()
150 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen6_emit_breadcrumb_rcs()
151 *cs++ = PIPE_CONTROL_QW_WRITE; in gen6_emit_breadcrumb_rcs()
152 *cs++ = intel_gt_scratch_offset(rq->engine->gt, in gen6_emit_breadcrumb_rcs()
155 *cs++ = 0; in gen6_emit_breadcrumb_rcs()
158 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen6_emit_breadcrumb_rcs()
159 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | in gen6_emit_breadcrumb_rcs()
164 *cs++ = i915_request_active_timeline(rq)->hwsp_offset | in gen6_emit_breadcrumb_rcs()
166 *cs++ = rq->fence.seqno; in gen6_emit_breadcrumb_rcs()
168 *cs++ = MI_USER_INTERRUPT; in gen6_emit_breadcrumb_rcs()
169 *cs++ = MI_NOOP; in gen6_emit_breadcrumb_rcs()
171 rq->tail = intel_ring_offset(rq, cs); in gen6_emit_breadcrumb_rcs()
172 assert_ring_tail_valid(rq->ring, rq->tail); in gen6_emit_breadcrumb_rcs()
174 return cs; in gen6_emit_breadcrumb_rcs()
179 u32 cmd, *cs; in mi_flush_dw() local
181 cs = intel_ring_begin(rq, 4); in mi_flush_dw()
182 if (IS_ERR(cs)) in mi_flush_dw()
183 return PTR_ERR(cs); in mi_flush_dw()
196 * Bspec vol 1c.3 - blitter engine command streamer: in mi_flush_dw()
199 * Post-Sync Operation field is a value of 1h or 3h." in mi_flush_dw()
203 *cs++ = cmd; in mi_flush_dw()
204 *cs++ = HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; in mi_flush_dw()
205 *cs++ = 0; in mi_flush_dw()
206 *cs++ = MI_NOOP; in mi_flush_dw()
208 intel_ring_advance(rq, cs); in mi_flush_dw()
233 u32 *cs; in gen6_emit_bb_start() local
239 cs = intel_ring_begin(rq, 2); in gen6_emit_bb_start()
240 if (IS_ERR(cs)) in gen6_emit_bb_start()
241 return PTR_ERR(cs); in gen6_emit_bb_start()
243 cs = __gen6_emit_bb_start(cs, offset, security); in gen6_emit_bb_start()
244 intel_ring_advance(rq, cs); in gen6_emit_bb_start()
255 u32 *cs; in hsw_emit_bb_start() local
261 cs = intel_ring_begin(rq, 2); in hsw_emit_bb_start()
262 if (IS_ERR(cs)) in hsw_emit_bb_start()
263 return PTR_ERR(cs); in hsw_emit_bb_start()
265 cs = __gen6_emit_bb_start(cs, offset, security); in hsw_emit_bb_start()
266 intel_ring_advance(rq, cs); in hsw_emit_bb_start()
273 u32 *cs; in gen7_stall_cs() local
275 cs = intel_ring_begin(rq, 4); in gen7_stall_cs()
276 if (IS_ERR(cs)) in gen7_stall_cs()
277 return PTR_ERR(cs); in gen7_stall_cs()
279 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen7_stall_cs()
280 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; in gen7_stall_cs()
281 *cs++ = 0; in gen7_stall_cs()
282 *cs++ = 0; in gen7_stall_cs()
283 intel_ring_advance(rq, cs); in gen7_stall_cs()
291 intel_gt_scratch_offset(rq->engine->gt, in gen7_emit_flush_rcs()
293 u32 *cs, flags = 0; in gen7_emit_flush_rcs() local
300 * read-cache invalidate bits set) must have the CS_STALL bit set. We in gen7_emit_flush_rcs()
306 * CS_STALL suggests at least a post-sync write. in gen7_emit_flush_rcs()
332 * Workaround: we must issue a pipe_control with CS-stall bit in gen7_emit_flush_rcs()
339 cs = intel_ring_begin(rq, 4); in gen7_emit_flush_rcs()
340 if (IS_ERR(cs)) in gen7_emit_flush_rcs()
341 return PTR_ERR(cs); in gen7_emit_flush_rcs()
343 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen7_emit_flush_rcs()
344 *cs++ = flags; in gen7_emit_flush_rcs()
345 *cs++ = scratch_addr; in gen7_emit_flush_rcs()
346 *cs++ = 0; in gen7_emit_flush_rcs()
347 intel_ring_advance(rq, cs); in gen7_emit_flush_rcs()
352 u32 *gen7_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen7_emit_breadcrumb_rcs() argument
354 *cs++ = GFX_OP_PIPE_CONTROL(4); in gen7_emit_breadcrumb_rcs()
355 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | in gen7_emit_breadcrumb_rcs()
362 *cs++ = i915_request_active_timeline(rq)->hwsp_offset; in gen7_emit_breadcrumb_rcs()
363 *cs++ = rq->fence.seqno; in gen7_emit_breadcrumb_rcs()
365 *cs++ = MI_USER_INTERRUPT; in gen7_emit_breadcrumb_rcs()
366 *cs++ = MI_NOOP; in gen7_emit_breadcrumb_rcs()
368 rq->tail = intel_ring_offset(rq, cs); in gen7_emit_breadcrumb_rcs()
369 assert_ring_tail_valid(rq->ring, rq->tail); in gen7_emit_breadcrumb_rcs()
371 return cs; in gen7_emit_breadcrumb_rcs()
374 u32 *gen6_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen6_emit_breadcrumb_xcs() argument
376 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); in gen6_emit_breadcrumb_xcs()
377 …GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADD… in gen6_emit_breadcrumb_xcs()
379 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; in gen6_emit_breadcrumb_xcs()
380 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; in gen6_emit_breadcrumb_xcs()
381 *cs++ = rq->fence.seqno; in gen6_emit_breadcrumb_xcs()
383 *cs++ = MI_USER_INTERRUPT; in gen6_emit_breadcrumb_xcs()
385 rq->tail = intel_ring_offset(rq, cs); in gen6_emit_breadcrumb_xcs()
386 assert_ring_tail_valid(rq->ring, rq->tail); in gen6_emit_breadcrumb_xcs()
388 return cs; in gen6_emit_breadcrumb_xcs()
392 u32 *gen7_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen7_emit_breadcrumb_xcs() argument
396 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); in gen7_emit_breadcrumb_xcs()
397 …GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADD… in gen7_emit_breadcrumb_xcs()
399 *cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB | in gen7_emit_breadcrumb_xcs()
401 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; in gen7_emit_breadcrumb_xcs()
402 *cs++ = rq->fence.seqno; in gen7_emit_breadcrumb_xcs()
405 *cs++ = MI_STORE_DWORD_INDEX; in gen7_emit_breadcrumb_xcs()
406 *cs++ = I915_GEM_HWS_SEQNO_ADDR; in gen7_emit_breadcrumb_xcs()
407 *cs++ = rq->fence.seqno; in gen7_emit_breadcrumb_xcs()
410 *cs++ = MI_FLUSH_DW; in gen7_emit_breadcrumb_xcs()
411 *cs++ = 0; in gen7_emit_breadcrumb_xcs()
412 *cs++ = 0; in gen7_emit_breadcrumb_xcs()
414 *cs++ = MI_USER_INTERRUPT; in gen7_emit_breadcrumb_xcs()
415 *cs++ = MI_NOOP; in gen7_emit_breadcrumb_xcs()
417 rq->tail = intel_ring_offset(rq, cs); in gen7_emit_breadcrumb_xcs()
418 assert_ring_tail_valid(rq->ring, rq->tail); in gen7_emit_breadcrumb_xcs()
420 return cs; in gen7_emit_breadcrumb_xcs()
427 ~(engine->irq_enable_mask | engine->irq_keep_mask)); in gen6_irq_enable()
432 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); in gen6_irq_enable()
437 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask); in gen6_irq_disable()
438 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); in gen6_irq_disable()
443 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask); in hsw_irq_enable_vecs()
448 gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask); in hsw_irq_enable_vecs()
454 gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask); in hsw_irq_disable_vecs()