Lines Matching +full:cs +full:- +full:1
1 // SPDX-License-Identifier: MIT
24 * The per-platform tables are u8-encoded in @data. Decode @data and set the
29 * [7]: create NOPs - number of NOPs are set in lower bits
60 const u32 base = engine->mmio_base; in set_offsets()
78 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
95 } while (--count); in set_offsets()
101 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
107 NOP(1),
142 NOP(1),
172 LRI(1, POSTED),
226 NOP(1),
258 NOP(1),
276 NOP(1),
292 NOP(1),
322 LRI(1, 0),
329 NOP(1),
359 LRI(1, 0),
413 NOP(1),
431 NOP(1),
443 LRI(1, POSTED),
447 LRI(1, 0),
454 NOP(1),
488 LRI(1, 0),
490 NOP(3 + 9 + 1),
544 NOP(1),
550 NOP(1),
584 LRI(1, 0),
591 NOP(1),
609 NOP(1),
627 LRI(1, 0),
634 NOP(1),
652 NOP(1),
670 LRI(1, 0),
690 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && in reg_offsets()
693 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { in reg_offsets()
694 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) in reg_offsets()
696 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
698 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in reg_offsets()
700 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
702 else if (GRAPHICS_VER(engine->i915) >= 11) in reg_offsets()
704 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
709 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
711 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
713 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
722 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_mi_mode()
724 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_mi_mode()
726 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_mi_mode()
728 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
731 return -1; in lrc_ring_mi_mode()
736 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_bb_offset()
738 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_bb_offset()
740 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_bb_offset()
742 else if (GRAPHICS_VER(engine->i915) >= 8 && in lrc_ring_bb_offset()
743 engine->class == RENDER_CLASS) in lrc_ring_bb_offset()
746 return -1; in lrc_ring_bb_offset()
751 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_gpr0()
753 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_gpr0()
755 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_gpr0()
757 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
760 return -1; in lrc_ring_gpr0()
765 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
767 else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
770 return -1; in lrc_ring_wa_bb_per_ctx()
798 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_cmd_buf_cctl()
804 else if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
805 return -1; in lrc_ring_cmd_buf_cctl()
806 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
808 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
811 return -1; in lrc_ring_cmd_buf_cctl()
817 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_indirect_offset_default()
819 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_indirect_offset_default()
821 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_indirect_offset_default()
823 else if (GRAPHICS_VER(engine->i915) >= 8) in lrc_ring_indirect_offset_default()
826 GEM_BUG_ON(GRAPHICS_VER(engine->i915) < 8); in lrc_ring_indirect_offset_default()
836 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in lrc_setup_bb_per_ctx()
837 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = in lrc_setup_bb_per_ctx()
851 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_setup_indirect_ctx()
852 regs[lrc_ring_indirect_ptr(engine) + 1] = in lrc_setup_indirect_ctx()
855 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_setup_indirect_ctx()
856 regs[lrc_ring_indirect_offset(engine) + 1] = in lrc_setup_indirect_ctx()
867 * the LRC run-alone bit or else the encryption will not happen. in ctx_needs_runalone()
869 if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 70) && in ctx_needs_runalone()
870 (ce->engine->class == COMPUTE_CLASS || ce->engine->class == RENDER_CLASS)) { in ctx_needs_runalone()
872 gem_ctx = rcu_dereference(ce->gem_context); in ctx_needs_runalone()
874 ctx_is_protected = gem_ctx->uses_protected_content; in ctx_needs_runalone()
893 if (GRAPHICS_VER(engine->i915) < 11) in init_common_regs()
900 regs[CTX_TIMESTAMP] = ce->stats.runtime.last; in init_common_regs()
903 if (loc != -1) in init_common_regs()
904 regs[loc + 1] = 0; in init_common_regs()
910 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_regs()
912 if (wa_ctx->per_ctx.size) { in init_wa_bb_regs()
913 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); in init_wa_bb_regs()
915 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_regs()
916 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = in init_wa_bb_regs()
917 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; in init_wa_bb_regs()
920 if (wa_ctx->indirect_ctx.size) { in init_wa_bb_regs()
922 i915_ggtt_offset(wa_ctx->vma) + in init_wa_bb_regs()
923 wa_ctx->indirect_ctx.offset, in init_wa_bb_regs()
924 wa_ctx->indirect_ctx.size); in init_wa_bb_regs()
930 if (i915_vm_is_4lvl(&ppgtt->vm)) { in init_ppgtt_regs()
939 ASSIGN_CTX_PDP(ppgtt, regs, 1); in init_ppgtt_regs()
947 return i915_vm_to_ggtt(vm)->alias; in vm_alias()
957 if (x != -1) { in __reset_stop_ring()
958 regs[x + 1] &= ~STOP_RING; in __reset_stop_ring()
959 regs[x + 1] |= STOP_RING << 16; in __reset_stop_ring()
985 init_ppgtt_regs(regs, vm_alias(ce->vm)); in __lrc_init_regs()
996 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); in lrc_init_regs()
1002 __reset_stop_ring(ce->lrc_reg_state, engine); in lrc_reset_regs()
1011 vaddr += engine->context_size; in set_redzone()
1022 vaddr += engine->context_size; in check_redzone()
1025 drm_err_once(&engine->i915->drm, in check_redzone()
1027 engine->name); in check_redzone()
1032 return PAGE_SIZE * ce->wa_bb_page; in context_wa_bb_offset()
1045 GEM_BUG_ON(!ce->wa_bb_page); in context_wabb()
1047 ptr = ce->lrc_reg_state; in context_wabb()
1048 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ in context_wabb()
1063 if (engine->default_state) { in lrc_init_state()
1064 shmem_read(engine->default_state, 0, in lrc_init_state()
1065 state, engine->context_size); in lrc_init_state()
1066 __set_bit(CONTEXT_VALID_BIT, &ce->flags); in lrc_init_state()
1070 /* Clear the ppHWSP (inc. per-context counters) */ in lrc_init_state()
1074 if (ce->wa_bb_page) in lrc_init_state()
1086 return i915_ggtt_offset(ce->state) + context_wa_bb_offset(ce); in lrc_indirect_bb()
1089 static u32 *setup_predicate_disable_wa(const struct intel_context *ce, u32 *cs) in setup_predicate_disable_wa() argument
1092 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1093 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1094 *cs++ = 0; in setup_predicate_disable_wa()
1095 *cs++ = 0; /* No predication */ in setup_predicate_disable_wa()
1098 *cs++ = MI_BATCH_BUFFER_END | BIT(15); in setup_predicate_disable_wa()
1099 *cs++ = MI_SET_PREDICATE | MI_SET_PREDICATE_DISABLE; in setup_predicate_disable_wa()
1102 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1103 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1104 *cs++ = 0; in setup_predicate_disable_wa()
1105 *cs++ = 1; /* enable predication before the next BB */ in setup_predicate_disable_wa()
1107 *cs++ = MI_BATCH_BUFFER_END; in setup_predicate_disable_wa()
1108 GEM_BUG_ON(offset_in_page(cs) > DG2_PREDICATE_RESULT_WA); in setup_predicate_disable_wa()
1110 return cs; in setup_predicate_disable_wa()
1120 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __lrc_alloc_state()
1125 if (GRAPHICS_VER(engine->i915) >= 12) { in __lrc_alloc_state()
1126 ce->wa_bb_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1132 ce->parallel.guc.parent_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1136 obj = i915_gem_object_create_lmem(engine->i915, context_size, in __lrc_alloc_state()
1139 obj = i915_gem_object_create_shmem(engine->i915, context_size); in __lrc_alloc_state()
1148 if (intel_gt_needs_wa_22016122933(engine->gt)) in __lrc_alloc_state()
1152 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in __lrc_alloc_state()
1164 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); in pinned_timeline()
1175 GEM_BUG_ON(ce->state); in lrc_alloc()
1181 ring = intel_engine_create_ring(engine, ce->ring_size); in lrc_alloc()
1187 if (!page_mask_bits(ce->timeline)) { in lrc_alloc()
1194 if (unlikely(ce->timeline)) in lrc_alloc()
1197 tl = intel_timeline_create(engine->gt); in lrc_alloc()
1203 ce->timeline = tl; in lrc_alloc()
1206 ce->ring = ring; in lrc_alloc()
1207 ce->state = vma; in lrc_alloc()
1222 intel_ring_reset(ce->ring, ce->ring->emit); in lrc_reset()
1225 lrc_init_regs(ce, ce->engine, true); in lrc_reset()
1226 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); in lrc_reset()
1235 GEM_BUG_ON(!ce->state); in lrc_pre_pin()
1236 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); in lrc_pre_pin()
1238 *vaddr = i915_gem_object_pin_map(ce->state->obj, in lrc_pre_pin()
1239 intel_gt_coherent_map_type(ce->engine->gt, in lrc_pre_pin()
1240 ce->state->obj, in lrc_pre_pin()
1252 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; in lrc_pin()
1254 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) in lrc_pin()
1257 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); in lrc_pin()
1263 if (unlikely(ce->parallel.last_rq)) { in lrc_unpin()
1264 i915_request_put(ce->parallel.last_rq); in lrc_unpin()
1265 ce->parallel.last_rq = NULL; in lrc_unpin()
1267 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, in lrc_unpin()
1268 ce->engine); in lrc_unpin()
1273 i915_gem_object_unpin_map(ce->state->obj); in lrc_post_unpin()
1278 if (!ce->state) in lrc_fini()
1281 intel_ring_put(fetch_and_zero(&ce->ring)); in lrc_fini()
1282 i915_vma_put(fetch_and_zero(&ce->state)); in lrc_fini()
1289 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); in lrc_destroy()
1299 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_timestamp_wa() argument
1301 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_timestamp_wa()
1304 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1305 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_timestamp_wa()
1307 *cs++ = 0; in gen12_emit_timestamp_wa()
1309 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1312 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1313 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1315 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1318 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1319 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1321 return cs; in gen12_emit_timestamp_wa()
1325 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) in gen12_emit_restore_scratch() argument
1327 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
1329 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_restore_scratch()
1332 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_restore_scratch()
1333 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_restore_scratch()
1334 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
1335 *cs++ = 0; in gen12_emit_restore_scratch()
1337 return cs; in gen12_emit_restore_scratch()
1341 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_cmd_buf_wa() argument
1343 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
1345 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_cmd_buf_wa()
1348 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1349 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_cmd_buf_wa()
1350 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
1351 *cs++ = 0; in gen12_emit_cmd_buf_wa()
1353 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_cmd_buf_wa()
1356 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1357 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); in gen12_emit_cmd_buf_wa()
1359 return cs; in gen12_emit_cmd_buf_wa()
1370 dg2_emit_draw_watermark_setting(u32 *cs) in dg2_emit_draw_watermark_setting() argument
1372 *cs++ = MI_LOAD_REGISTER_IMM(1); in dg2_emit_draw_watermark_setting()
1373 *cs++ = i915_mmio_reg_offset(DRAW_WATERMARK); in dg2_emit_draw_watermark_setting()
1374 *cs++ = REG_FIELD_PREP(VERT_WM_VAL, 0x3FF); in dg2_emit_draw_watermark_setting()
1376 return cs; in dg2_emit_draw_watermark_setting()
1380 gen12_invalidate_state_cache(u32 *cs) in gen12_invalidate_state_cache() argument
1382 *cs++ = MI_LOAD_REGISTER_IMM(1); in gen12_invalidate_state_cache()
1383 *cs++ = i915_mmio_reg_offset(GEN12_CS_DEBUG_MODE2); in gen12_invalidate_state_cache()
1384 *cs++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); in gen12_invalidate_state_cache()
1385 return cs; in gen12_invalidate_state_cache()
1389 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_rcs() argument
1391 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1392 cs = gen12_emit_cmd_buf_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1393 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_rcs()
1396 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1397 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); in gen12_emit_indirect_ctx_rcs()
1399 cs = gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_rcs()
1402 if (IS_GFX_GT_IP_RANGE(ce->engine->gt, IP_VER(12, 0), IP_VER(12, 10))) in gen12_emit_indirect_ctx_rcs()
1403 cs = gen12_invalidate_state_cache(cs); in gen12_emit_indirect_ctx_rcs()
1406 if (IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1407 IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 71), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1408 IS_DG2(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1409 cs = dg2_emit_draw_watermark_setting(cs); in gen12_emit_indirect_ctx_rcs()
1411 return cs; in gen12_emit_indirect_ctx_rcs()
1415 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_xcs() argument
1417 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_xcs()
1418 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_xcs()
1421 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_xcs()
1422 if (ce->engine->class == COMPUTE_CLASS) in gen12_emit_indirect_ctx_xcs()
1423 cs = gen8_emit_pipe_control(cs, in gen12_emit_indirect_ctx_xcs()
1427 return gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_xcs()
1430 static u32 *xehp_emit_fastcolor_blt_wabb(const struct intel_context *ce, u32 *cs) in xehp_emit_fastcolor_blt_wabb() argument
1432 struct intel_gt *gt = ce->engine->gt; in xehp_emit_fastcolor_blt_wabb()
1433 int mocs = gt->mocs.uc_index << 1; in xehp_emit_fastcolor_blt_wabb()
1443 * BG0 -> 5100000E in xehp_emit_fastcolor_blt_wabb()
1444 * BG1 -> 0000003F (Dest pitch) in xehp_emit_fastcolor_blt_wabb()
1445 * BG2 -> 00000000 (X1, Y1) = (0, 0) in xehp_emit_fastcolor_blt_wabb()
1446 * BG3 -> 00040001 (X2, Y2) = (1, 4) in xehp_emit_fastcolor_blt_wabb()
1447 * BG4 -> scratch in xehp_emit_fastcolor_blt_wabb()
1448 * BG5 -> scratch in xehp_emit_fastcolor_blt_wabb()
1449 * BG6-12 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1450 * BG13 -> 20004004 (Surf. Width= 2,Surf. Height = 5 ) in xehp_emit_fastcolor_blt_wabb()
1451 * BG14 -> 00000010 (Qpitch = 4) in xehp_emit_fastcolor_blt_wabb()
1452 * BG15 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1454 *cs++ = XY_FAST_COLOR_BLT_CMD | (16 - 2); in xehp_emit_fastcolor_blt_wabb()
1455 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | 0x3f; in xehp_emit_fastcolor_blt_wabb()
1456 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1457 *cs++ = 4 << 16 | 1; in xehp_emit_fastcolor_blt_wabb()
1458 *cs++ = lower_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1459 *cs++ = upper_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1460 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1461 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1462 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1463 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1464 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1465 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1466 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1467 *cs++ = 0x20004004; in xehp_emit_fastcolor_blt_wabb()
1468 *cs++ = 0x10; in xehp_emit_fastcolor_blt_wabb()
1469 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1471 return cs; in xehp_emit_fastcolor_blt_wabb()
1475 xehp_emit_per_ctx_bb(const struct intel_context *ce, u32 *cs) in xehp_emit_per_ctx_bb() argument
1478 if (NEEDS_FASTCOLOR_BLT_WABB(ce->engine)) in xehp_emit_per_ctx_bb()
1479 cs = xehp_emit_fastcolor_blt_wabb(ce, cs); in xehp_emit_per_ctx_bb()
1481 return cs; in xehp_emit_per_ctx_bb()
1491 u32 *cs; in setup_per_ctx_bb() local
1493 cs = emit(ce, start); in setup_per_ctx_bb()
1496 *cs++ = MI_BATCH_BUFFER_END; in setup_per_ctx_bb()
1498 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_per_ctx_bb()
1499 lrc_setup_bb_per_ctx(ce->lrc_reg_state, engine, in setup_per_ctx_bb()
1509 u32 *cs; in setup_indirect_ctx_bb() local
1511 cs = emit(ce, start); in setup_indirect_ctx_bb()
1512 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_indirect_ctx_bb()
1513 while ((unsigned long)cs % CACHELINE_BYTES) in setup_indirect_ctx_bb()
1514 *cs++ = MI_NOOP; in setup_indirect_ctx_bb()
1516 GEM_BUG_ON(cs - start > DG2_PREDICATE_RESULT_BB / sizeof(*start)); in setup_indirect_ctx_bb()
1519 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
1521 (cs - start) * sizeof(*cs)); in setup_indirect_ctx_bb()
1532 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template)
1533 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
1534 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC)
1535 * bits 53-54: mbz, reserved for use by hardware
1536 * bits 55-63: group ID, currently unused and set to 0
1540 * bits 32-36: reserved
1541 * bits 37-47: SW context ID
1544 * bits 55-60: SW counter
1545 * bits 61-63: engine class
1549 * bits 32-37: virtual function number
1551 * bits 39-54: SW context ID
1552 * bits 55-57: reserved
1553 * bits 58-63: SW counter
1563 if (i915_vm_is_4lvl(ce->vm)) in lrc_descriptor()
1568 if (GRAPHICS_VER(ce->vm->i915) == 8) in lrc_descriptor()
1571 return i915_ggtt_offset(ce->state) | desc; in lrc_descriptor()
1578 struct intel_ring *ring = ce->ring; in lrc_update_regs()
1579 u32 *regs = ce->lrc_reg_state; in lrc_update_regs()
1582 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); in lrc_update_regs()
1584 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_update_regs()
1586 regs[CTX_RING_TAIL] = ring->tail; in lrc_update_regs()
1587 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_update_regs()
1590 if (engine->class == RENDER_CLASS) { in lrc_update_regs()
1592 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in lrc_update_regs()
1597 if (ce->wa_bb_page) { in lrc_update_regs()
1598 u32 *(*fn)(const struct intel_context *ce, u32 *cs); in lrc_update_regs()
1601 if (ce->engine->class == RENDER_CLASS) in lrc_update_regs()
1605 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in lrc_update_regs()
1616 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); in lrc_update_offsets()
1623 const struct intel_ring *ring = ce->ring; in lrc_check_regs()
1624 u32 *regs = ce->lrc_reg_state; in lrc_check_regs()
1628 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { in lrc_check_regs()
1630 engine->name, in lrc_check_regs()
1632 i915_ggtt_offset(ring->vma)); in lrc_check_regs()
1633 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_check_regs()
1638 (RING_CTL_SIZE(ring->size) | RING_VALID)) { in lrc_check_regs()
1640 engine->name, in lrc_check_regs()
1642 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); in lrc_check_regs()
1643 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_check_regs()
1648 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { in lrc_check_regs()
1650 engine->name, regs[x + 1]); in lrc_check_regs()
1651 regs[x + 1] &= ~STOP_RING; in lrc_check_regs()
1652 regs[x + 1] |= STOP_RING << 16; in lrc_check_regs()
1668 * it for a short period and this batch in non-premptible. We can ofcourse
1681 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1685 *batch++ = MI_LOAD_REGISTER_IMM(1); in gen8_emit_flush_coherentl3_wa()
1696 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1724 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
1762 *batch++ = i915_mmio_reg_offset(lri->reg); in emit_lri()
1763 *batch++ = lri->value; in emit_lri()
1764 } while (lri++, --count); in emit_lri()
1811 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
1850 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); in lrc_create_wa_ctx()
1854 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_create_wa_ctx()
1860 engine->wa_ctx.vma = vma; in lrc_create_wa_ctx()
1870 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_fini_wa_ctx()
1877 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in lrc_init_wa_ctx()
1879 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx in lrc_init_wa_ctx()
1887 if (GRAPHICS_VER(engine->i915) >= 11 || in lrc_init_wa_ctx()
1888 !(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in lrc_init_wa_ctx()
1891 if (GRAPHICS_VER(engine->i915) == 9) { in lrc_init_wa_ctx()
1893 wa_bb_fn[1] = NULL; in lrc_init_wa_ctx()
1894 } else if (GRAPHICS_VER(engine->i915) == 8) { in lrc_init_wa_ctx()
1896 wa_bb_fn[1] = NULL; in lrc_init_wa_ctx()
1906 drm_err(&engine->i915->drm, in lrc_init_wa_ctx()
1912 if (!engine->wa_ctx.vma) in lrc_init_wa_ctx()
1917 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); in lrc_init_wa_ctx()
1919 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); in lrc_init_wa_ctx()
1923 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); in lrc_init_wa_ctx()
1936 wa_bb[i]->offset = batch_ptr - batch; in lrc_init_wa_ctx()
1937 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, in lrc_init_wa_ctx()
1939 err = -EINVAL; in lrc_init_wa_ctx()
1944 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); in lrc_init_wa_ctx()
1946 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); in lrc_init_wa_ctx()
1948 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); in lrc_init_wa_ctx()
1949 __i915_gem_object_release_map(wa_ctx->vma->obj); in lrc_init_wa_ctx()
1953 err = i915_inject_probe_error(engine->i915, -ENODEV); in lrc_init_wa_ctx()
1957 i915_vma_unpin(wa_ctx->vma); in lrc_init_wa_ctx()
1959 if (err == -EDEADLK) { in lrc_init_wa_ctx()
1967 i915_vma_put(engine->wa_ctx.vma); in lrc_init_wa_ctx()
1977 stats->runtime.num_underflow++; in st_runtime_underflow()
1978 stats->runtime.max_underflow = in st_runtime_underflow()
1979 max_t(u32, stats->runtime.max_underflow, -dt); in st_runtime_underflow()
1991 return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); in lrc_get_runtime()
1996 struct intel_context_stats *stats = &ce->stats; in lrc_update_runtime()
2000 old = stats->runtime.last; in lrc_update_runtime()
2001 stats->runtime.last = lrc_get_runtime(ce); in lrc_update_runtime()
2002 dt = stats->runtime.last - old; in lrc_update_runtime()
2008 old, stats->runtime.last, dt); in lrc_update_runtime()
2013 ewma_runtime_add(&stats->runtime.avg, dt); in lrc_update_runtime()
2014 stats->runtime.total += dt; in lrc_update_runtime()