Lines Matching +full:cs +full:- +full:x
1 // SPDX-License-Identifier: MIT
24 * The per-platform tables are u8-encoded in @data. Decode @data and set the
29 * [7]: create NOPs - number of NOPs are set in lower bits
51 #define NOP(x) (BIT(7) | (x)) in set_offsets() argument
54 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) in set_offsets() argument
55 #define REG16(x) \ in set_offsets() argument
56 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ in set_offsets()
57 (((x) >> 2) & 0x7f) in set_offsets()
60 const u32 base = engine->mmio_base; in set_offsets()
78 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
95 } while (--count); in set_offsets()
101 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
690 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && in reg_offsets()
693 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { in reg_offsets()
694 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) in reg_offsets()
696 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
698 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in reg_offsets()
700 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
702 else if (GRAPHICS_VER(engine->i915) >= 11) in reg_offsets()
704 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
709 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
711 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
713 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
722 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_mi_mode()
724 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_mi_mode()
726 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_mi_mode()
728 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
731 return -1; in lrc_ring_mi_mode()
736 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_bb_offset()
738 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_bb_offset()
740 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_bb_offset()
742 else if (GRAPHICS_VER(engine->i915) >= 8 && in lrc_ring_bb_offset()
743 engine->class == RENDER_CLASS) in lrc_ring_bb_offset()
746 return -1; in lrc_ring_bb_offset()
751 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_gpr0()
753 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_gpr0()
755 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_gpr0()
757 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
760 return -1; in lrc_ring_gpr0()
765 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
767 else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
770 return -1; in lrc_ring_wa_bb_per_ctx()
775 int x; in lrc_ring_indirect_ptr() local
777 x = lrc_ring_wa_bb_per_ctx(engine); in lrc_ring_indirect_ptr()
778 if (x < 0) in lrc_ring_indirect_ptr()
779 return x; in lrc_ring_indirect_ptr()
781 return x + 2; in lrc_ring_indirect_ptr()
786 int x; in lrc_ring_indirect_offset() local
788 x = lrc_ring_indirect_ptr(engine); in lrc_ring_indirect_offset()
789 if (x < 0) in lrc_ring_indirect_offset()
790 return x; in lrc_ring_indirect_offset()
792 return x + 2; in lrc_ring_indirect_offset()
798 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_cmd_buf_cctl()
804 else if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
805 return -1; in lrc_ring_cmd_buf_cctl()
806 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
808 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
811 return -1; in lrc_ring_cmd_buf_cctl()
817 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_indirect_offset_default()
819 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_indirect_offset_default()
821 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_indirect_offset_default()
823 else if (GRAPHICS_VER(engine->i915) >= 8) in lrc_ring_indirect_offset_default()
826 GEM_BUG_ON(GRAPHICS_VER(engine->i915) < 8); in lrc_ring_indirect_offset_default()
836 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in lrc_setup_bb_per_ctx()
851 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_setup_indirect_ctx()
855 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_setup_indirect_ctx()
867 * the LRC run-alone bit or else the encryption will not happen. in ctx_needs_runalone()
869 if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 70) && in ctx_needs_runalone()
870 (ce->engine->class == COMPUTE_CLASS || ce->engine->class == RENDER_CLASS)) { in ctx_needs_runalone()
872 gem_ctx = rcu_dereference(ce->gem_context); in ctx_needs_runalone()
874 ctx_is_protected = gem_ctx->uses_protected_content; in ctx_needs_runalone()
893 if (GRAPHICS_VER(engine->i915) < 11) in init_common_regs()
900 regs[CTX_TIMESTAMP] = ce->stats.runtime.last; in init_common_regs()
903 if (loc != -1) in init_common_regs()
910 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_regs()
912 if (wa_ctx->per_ctx.size) { in init_wa_bb_regs()
913 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); in init_wa_bb_regs()
915 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_regs()
917 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; in init_wa_bb_regs()
920 if (wa_ctx->indirect_ctx.size) { in init_wa_bb_regs()
922 i915_ggtt_offset(wa_ctx->vma) + in init_wa_bb_regs()
923 wa_ctx->indirect_ctx.offset, in init_wa_bb_regs()
924 wa_ctx->indirect_ctx.size); in init_wa_bb_regs()
930 if (i915_vm_is_4lvl(&ppgtt->vm)) { in init_ppgtt_regs()
947 return i915_vm_to_ggtt(vm)->alias; in vm_alias()
954 int x; in __reset_stop_ring() local
956 x = lrc_ring_mi_mode(engine); in __reset_stop_ring()
957 if (x != -1) { in __reset_stop_ring()
958 regs[x + 1] &= ~STOP_RING; in __reset_stop_ring()
959 regs[x + 1] |= STOP_RING << 16; in __reset_stop_ring()
985 init_ppgtt_regs(regs, vm_alias(ce->vm)); in __lrc_init_regs()
996 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); in lrc_init_regs()
1002 __reset_stop_ring(ce->lrc_reg_state, engine); in lrc_reset_regs()
1011 vaddr += engine->context_size; in set_redzone()
1022 vaddr += engine->context_size; in check_redzone()
1025 drm_err_once(&engine->i915->drm, in check_redzone()
1027 engine->name); in check_redzone()
1032 return PAGE_SIZE * ce->wa_bb_page; in context_wa_bb_offset()
1045 GEM_BUG_ON(!ce->wa_bb_page); in context_wabb()
1047 ptr = ce->lrc_reg_state; in context_wabb()
1048 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ in context_wabb()
1063 if (engine->default_state) { in lrc_init_state()
1064 shmem_read(engine->default_state, 0, in lrc_init_state()
1065 state, engine->context_size); in lrc_init_state()
1066 __set_bit(CONTEXT_VALID_BIT, &ce->flags); in lrc_init_state()
1070 /* Clear the ppHWSP (inc. per-context counters) */ in lrc_init_state()
1074 if (ce->wa_bb_page) in lrc_init_state()
1086 return i915_ggtt_offset(ce->state) + context_wa_bb_offset(ce); in lrc_indirect_bb()
1089 static u32 *setup_predicate_disable_wa(const struct intel_context *ce, u32 *cs) in setup_predicate_disable_wa() argument
1092 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1093 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1094 *cs++ = 0; in setup_predicate_disable_wa()
1095 *cs++ = 0; /* No predication */ in setup_predicate_disable_wa()
1098 *cs++ = MI_BATCH_BUFFER_END | BIT(15); in setup_predicate_disable_wa()
1099 *cs++ = MI_SET_PREDICATE | MI_SET_PREDICATE_DISABLE; in setup_predicate_disable_wa()
1102 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1103 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1104 *cs++ = 0; in setup_predicate_disable_wa()
1105 *cs++ = 1; /* enable predication before the next BB */ in setup_predicate_disable_wa()
1107 *cs++ = MI_BATCH_BUFFER_END; in setup_predicate_disable_wa()
1108 GEM_BUG_ON(offset_in_page(cs) > DG2_PREDICATE_RESULT_WA); in setup_predicate_disable_wa()
1110 return cs; in setup_predicate_disable_wa()
1120 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __lrc_alloc_state()
1125 if (GRAPHICS_VER(engine->i915) >= 12) { in __lrc_alloc_state()
1126 ce->wa_bb_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1132 ce->parallel.guc.parent_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1136 obj = i915_gem_object_create_lmem(engine->i915, context_size, in __lrc_alloc_state()
1139 obj = i915_gem_object_create_shmem(engine->i915, context_size); in __lrc_alloc_state()
1148 if (intel_gt_needs_wa_22016122933(engine->gt)) in __lrc_alloc_state()
1152 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in __lrc_alloc_state()
1164 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); in pinned_timeline()
1175 GEM_BUG_ON(ce->state); in lrc_alloc()
1181 ring = intel_engine_create_ring(engine, ce->ring_size); in lrc_alloc()
1187 if (!page_mask_bits(ce->timeline)) { in lrc_alloc()
1194 if (unlikely(ce->timeline)) in lrc_alloc()
1197 tl = intel_timeline_create(engine->gt); in lrc_alloc()
1203 ce->timeline = tl; in lrc_alloc()
1206 ce->ring = ring; in lrc_alloc()
1207 ce->state = vma; in lrc_alloc()
1222 intel_ring_reset(ce->ring, ce->ring->emit); in lrc_reset()
1225 lrc_init_regs(ce, ce->engine, true); in lrc_reset()
1226 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); in lrc_reset()
1235 GEM_BUG_ON(!ce->state); in lrc_pre_pin()
1236 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); in lrc_pre_pin()
1238 *vaddr = i915_gem_object_pin_map(ce->state->obj, in lrc_pre_pin()
1239 intel_gt_coherent_map_type(ce->engine->gt, in lrc_pre_pin()
1240 ce->state->obj, in lrc_pre_pin()
1252 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; in lrc_pin()
1254 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) in lrc_pin()
1257 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); in lrc_pin()
1263 if (unlikely(ce->parallel.last_rq)) { in lrc_unpin()
1264 i915_request_put(ce->parallel.last_rq); in lrc_unpin()
1265 ce->parallel.last_rq = NULL; in lrc_unpin()
1267 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, in lrc_unpin()
1268 ce->engine); in lrc_unpin()
1273 i915_gem_object_unpin_map(ce->state->obj); in lrc_post_unpin()
1278 if (!ce->state) in lrc_fini()
1281 intel_ring_put(fetch_and_zero(&ce->ring)); in lrc_fini()
1282 i915_vma_put(fetch_and_zero(&ce->state)); in lrc_fini()
1289 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); in lrc_destroy()
1299 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_timestamp_wa() argument
1301 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_timestamp_wa()
1304 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1305 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_timestamp_wa()
1307 *cs++ = 0; in gen12_emit_timestamp_wa()
1309 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1312 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1313 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1315 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1318 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1319 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1321 return cs; in gen12_emit_timestamp_wa()
1325 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) in gen12_emit_restore_scratch() argument
1327 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
1329 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_restore_scratch()
1332 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_restore_scratch()
1333 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_restore_scratch()
1334 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
1335 *cs++ = 0; in gen12_emit_restore_scratch()
1337 return cs; in gen12_emit_restore_scratch()
1341 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_cmd_buf_wa() argument
1343 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
1345 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_cmd_buf_wa()
1348 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1349 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_cmd_buf_wa()
1350 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
1351 *cs++ = 0; in gen12_emit_cmd_buf_wa()
1353 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_cmd_buf_wa()
1356 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1357 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); in gen12_emit_cmd_buf_wa()
1359 return cs; in gen12_emit_cmd_buf_wa()
1370 dg2_emit_draw_watermark_setting(u32 *cs) in dg2_emit_draw_watermark_setting() argument
1372 *cs++ = MI_LOAD_REGISTER_IMM(1); in dg2_emit_draw_watermark_setting()
1373 *cs++ = i915_mmio_reg_offset(DRAW_WATERMARK); in dg2_emit_draw_watermark_setting()
1374 *cs++ = REG_FIELD_PREP(VERT_WM_VAL, 0x3FF); in dg2_emit_draw_watermark_setting()
1376 return cs; in dg2_emit_draw_watermark_setting()
1380 gen12_invalidate_state_cache(u32 *cs) in gen12_invalidate_state_cache() argument
1382 *cs++ = MI_LOAD_REGISTER_IMM(1); in gen12_invalidate_state_cache()
1383 *cs++ = i915_mmio_reg_offset(GEN12_CS_DEBUG_MODE2); in gen12_invalidate_state_cache()
1384 *cs++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); in gen12_invalidate_state_cache()
1385 return cs; in gen12_invalidate_state_cache()
1389 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_rcs() argument
1391 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1392 cs = gen12_emit_cmd_buf_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1393 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_rcs()
1396 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1397 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); in gen12_emit_indirect_ctx_rcs()
1399 cs = gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_rcs()
1402 if (IS_GFX_GT_IP_RANGE(ce->engine->gt, IP_VER(12, 0), IP_VER(12, 10))) in gen12_emit_indirect_ctx_rcs()
1403 cs = gen12_invalidate_state_cache(cs); in gen12_emit_indirect_ctx_rcs()
1406 if (IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1407 IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 71), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1408 IS_DG2(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1409 cs = dg2_emit_draw_watermark_setting(cs); in gen12_emit_indirect_ctx_rcs()
1411 return cs; in gen12_emit_indirect_ctx_rcs()
1415 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_xcs() argument
1417 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_xcs()
1418 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_xcs()
1421 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_xcs()
1422 if (ce->engine->class == COMPUTE_CLASS) in gen12_emit_indirect_ctx_xcs()
1423 cs = gen8_emit_pipe_control(cs, in gen12_emit_indirect_ctx_xcs()
1427 return gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_xcs()
1430 static u32 *xehp_emit_fastcolor_blt_wabb(const struct intel_context *ce, u32 *cs) in xehp_emit_fastcolor_blt_wabb() argument
1432 struct intel_gt *gt = ce->engine->gt; in xehp_emit_fastcolor_blt_wabb()
1433 int mocs = gt->mocs.uc_index << 1; in xehp_emit_fastcolor_blt_wabb()
1443 * BG0 -> 5100000E in xehp_emit_fastcolor_blt_wabb()
1444 * BG1 -> 0000003F (Dest pitch) in xehp_emit_fastcolor_blt_wabb()
1445 * BG2 -> 00000000 (X1, Y1) = (0, 0) in xehp_emit_fastcolor_blt_wabb()
1446 * BG3 -> 00040001 (X2, Y2) = (1, 4) in xehp_emit_fastcolor_blt_wabb()
1447 * BG4 -> scratch in xehp_emit_fastcolor_blt_wabb()
1448 * BG5 -> scratch in xehp_emit_fastcolor_blt_wabb()
1449 * BG6-12 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1450 * BG13 -> 20004004 (Surf. Width= 2,Surf. Height = 5 ) in xehp_emit_fastcolor_blt_wabb()
1451 * BG14 -> 00000010 (Qpitch = 4) in xehp_emit_fastcolor_blt_wabb()
1452 * BG15 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1454 *cs++ = XY_FAST_COLOR_BLT_CMD | (16 - 2); in xehp_emit_fastcolor_blt_wabb()
1455 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | 0x3f; in xehp_emit_fastcolor_blt_wabb()
1456 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1457 *cs++ = 4 << 16 | 1; in xehp_emit_fastcolor_blt_wabb()
1458 *cs++ = lower_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1459 *cs++ = upper_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1460 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1461 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1462 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1463 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1464 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1465 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1466 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1467 *cs++ = 0x20004004; in xehp_emit_fastcolor_blt_wabb()
1468 *cs++ = 0x10; in xehp_emit_fastcolor_blt_wabb()
1469 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1471 return cs; in xehp_emit_fastcolor_blt_wabb()
1475 xehp_emit_per_ctx_bb(const struct intel_context *ce, u32 *cs) in xehp_emit_per_ctx_bb() argument
1478 if (NEEDS_FASTCOLOR_BLT_WABB(ce->engine)) in xehp_emit_per_ctx_bb()
1479 cs = xehp_emit_fastcolor_blt_wabb(ce, cs); in xehp_emit_per_ctx_bb()
1481 return cs; in xehp_emit_per_ctx_bb()
1491 u32 *cs; in setup_per_ctx_bb() local
1493 cs = emit(ce, start); in setup_per_ctx_bb()
1496 *cs++ = MI_BATCH_BUFFER_END; in setup_per_ctx_bb()
1498 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_per_ctx_bb()
1499 lrc_setup_bb_per_ctx(ce->lrc_reg_state, engine, in setup_per_ctx_bb()
1509 u32 *cs; in setup_indirect_ctx_bb() local
1511 cs = emit(ce, start); in setup_indirect_ctx_bb()
1512 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_indirect_ctx_bb()
1513 while ((unsigned long)cs % CACHELINE_BYTES) in setup_indirect_ctx_bb()
1514 *cs++ = MI_NOOP; in setup_indirect_ctx_bb()
1516 GEM_BUG_ON(cs - start > DG2_PREDICATE_RESULT_BB / sizeof(*start)); in setup_indirect_ctx_bb()
1519 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
1521 (cs - start) * sizeof(*cs)); in setup_indirect_ctx_bb()
1532 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template)
1533 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
1534 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC)
1535 * bits 53-54: mbz, reserved for use by hardware
1536 * bits 55-63: group ID, currently unused and set to 0
1540 * bits 32-36: reserved
1541 * bits 37-47: SW context ID
1544 * bits 55-60: SW counter
1545 * bits 61-63: engine class
1549 * bits 32-37: virtual function number
1551 * bits 39-54: SW context ID
1552 * bits 55-57: reserved
1553 * bits 58-63: SW counter
1563 if (i915_vm_is_4lvl(ce->vm)) in lrc_descriptor()
1568 if (GRAPHICS_VER(ce->vm->i915) == 8) in lrc_descriptor()
1571 return i915_ggtt_offset(ce->state) | desc; in lrc_descriptor()
1578 struct intel_ring *ring = ce->ring; in lrc_update_regs()
1579 u32 *regs = ce->lrc_reg_state; in lrc_update_regs()
1582 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); in lrc_update_regs()
1584 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_update_regs()
1586 regs[CTX_RING_TAIL] = ring->tail; in lrc_update_regs()
1587 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_update_regs()
1590 if (engine->class == RENDER_CLASS) { in lrc_update_regs()
1592 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in lrc_update_regs()
1597 if (ce->wa_bb_page) { in lrc_update_regs()
1598 u32 *(*fn)(const struct intel_context *ce, u32 *cs); in lrc_update_regs()
1601 if (ce->engine->class == RENDER_CLASS) in lrc_update_regs()
1605 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in lrc_update_regs()
1616 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); in lrc_update_offsets()
1623 const struct intel_ring *ring = ce->ring; in lrc_check_regs()
1624 u32 *regs = ce->lrc_reg_state; in lrc_check_regs()
1626 int x; in lrc_check_regs() local
1628 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { in lrc_check_regs()
1629 pr_err("%s: context submitted with incorrect RING_START [%08x], expected %08x\n", in lrc_check_regs()
1630 engine->name, in lrc_check_regs()
1632 i915_ggtt_offset(ring->vma)); in lrc_check_regs()
1633 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_check_regs()
1638 (RING_CTL_SIZE(ring->size) | RING_VALID)) { in lrc_check_regs()
1639 pr_err("%s: context submitted with incorrect RING_CTL [%08x], expected %08x\n", in lrc_check_regs()
1640 engine->name, in lrc_check_regs()
1642 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); in lrc_check_regs()
1643 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_check_regs()
1647 x = lrc_ring_mi_mode(engine); in lrc_check_regs()
1648 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { in lrc_check_regs()
1649 pr_err("%s: context submitted with STOP_RING [%08x] in RING_MI_MODE\n", in lrc_check_regs()
1650 engine->name, regs[x + 1]); in lrc_check_regs()
1651 regs[x + 1] &= ~STOP_RING; in lrc_check_regs()
1652 regs[x + 1] |= STOP_RING << 16; in lrc_check_regs()
1668 * it for a short period and this batch in non-premptible. We can ofcourse
1681 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1696 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1724 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
1762 *batch++ = i915_mmio_reg_offset(lri->reg); in emit_lri()
1763 *batch++ = lri->value; in emit_lri()
1764 } while (lri++, --count); in emit_lri()
1811 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
1850 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); in lrc_create_wa_ctx()
1854 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_create_wa_ctx()
1860 engine->wa_ctx.vma = vma; in lrc_create_wa_ctx()
1870 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_fini_wa_ctx()
1877 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in lrc_init_wa_ctx()
1879 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx in lrc_init_wa_ctx()
1887 if (GRAPHICS_VER(engine->i915) >= 11 || in lrc_init_wa_ctx()
1888 !(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in lrc_init_wa_ctx()
1891 if (GRAPHICS_VER(engine->i915) == 9) { in lrc_init_wa_ctx()
1894 } else if (GRAPHICS_VER(engine->i915) == 8) { in lrc_init_wa_ctx()
1906 drm_err(&engine->i915->drm, in lrc_init_wa_ctx()
1912 if (!engine->wa_ctx.vma) in lrc_init_wa_ctx()
1917 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); in lrc_init_wa_ctx()
1919 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); in lrc_init_wa_ctx()
1923 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); in lrc_init_wa_ctx()
1936 wa_bb[i]->offset = batch_ptr - batch; in lrc_init_wa_ctx()
1937 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, in lrc_init_wa_ctx()
1939 err = -EINVAL; in lrc_init_wa_ctx()
1944 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); in lrc_init_wa_ctx()
1946 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); in lrc_init_wa_ctx()
1948 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); in lrc_init_wa_ctx()
1949 __i915_gem_object_release_map(wa_ctx->vma->obj); in lrc_init_wa_ctx()
1953 err = i915_inject_probe_error(engine->i915, -ENODEV); in lrc_init_wa_ctx()
1957 i915_vma_unpin(wa_ctx->vma); in lrc_init_wa_ctx()
1959 if (err == -EDEADLK) { in lrc_init_wa_ctx()
1967 i915_vma_put(engine->wa_ctx.vma); in lrc_init_wa_ctx()
1977 stats->runtime.num_underflow++; in st_runtime_underflow()
1978 stats->runtime.max_underflow = in st_runtime_underflow()
1979 max_t(u32, stats->runtime.max_underflow, -dt); in st_runtime_underflow()
1991 return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); in lrc_get_runtime()
1996 struct intel_context_stats *stats = &ce->stats; in lrc_update_runtime()
2000 old = stats->runtime.last; in lrc_update_runtime()
2001 stats->runtime.last = lrc_get_runtime(ce); in lrc_update_runtime()
2002 dt = stats->runtime.last - old; in lrc_update_runtime()
2008 old, stats->runtime.last, dt); in lrc_update_runtime()
2013 ewma_runtime_add(&stats->runtime.avg, dt); in lrc_update_runtime()
2014 stats->runtime.total += dt; in lrc_update_runtime()