Lines Matching +full:engine +full:- +full:specific
1 // SPDX-License-Identifier: MIT
3 * Copyright © 2014-2018 Intel Corporation
30 * - Context workarounds: workarounds that touch registers that are
40 * - Engine workarounds: the list of these WAs is applied whenever the specific
41 * engine is reset. It's also possible that a set of engine classes share a
45 * driver is to tie those workarounds to the first compute/render engine that
46 * is registered. When executing with GuC submission, engine resets are
48 * written once, on engine initialization, and then passed to GuC, that
52 * Workarounds for registers specific to RCS and CCS should be implemented in
55 * xcs_engine_wa_init(). Workarounds for registers not belonging to a specific
56 * engine's MMIO range but that are part of of the common RCS/CCS reset domain
60 * - GT workarounds: the list of these WAs is applied whenever these registers
66 * - Register whitelist: some workarounds need to be implemented in userspace,
70 * these to/be-whitelisted registers to some special HW registers).
75 * - Workaround batchbuffers: buffers that get executed automatically by the
88 * engine registers are restored in a context restore sequence. This is
91 * - Other: There are WAs that, due to their nature, cannot be applied from a
103 wal->gt = gt; in wa_init_start()
104 wal->name = name; in wa_init_start()
105 wal->engine_name = engine_name; in wa_init_start()
113 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { in wa_init_finish()
114 struct i915_wa *list = kmemdup_array(wal->list, wal->count, in wa_init_finish()
118 kfree(wal->list); in wa_init_finish()
119 wal->list = list; in wa_init_finish()
123 if (!wal->count) in wa_init_finish()
126 gt_dbg(wal->gt, "Initialized %u %s workarounds on %s\n", in wa_init_finish()
127 wal->wa_count, wal->name, wal->engine_name); in wa_init_finish()
137 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wal_get_fw_for_rmw()
139 wa->reg, in wal_get_fw_for_rmw()
148 unsigned int addr = i915_mmio_reg_offset(wa->reg); in _wa_add()
149 struct drm_i915_private *i915 = wal->gt->i915; in _wa_add()
150 unsigned int start = 0, end = wal->count; in _wa_add()
156 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ in _wa_add()
159 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), in _wa_add()
162 drm_err(&i915->drm, "No space for workaround init!\n"); in _wa_add()
166 if (wal->list) { in _wa_add()
167 memcpy(list, wal->list, sizeof(*wa) * wal->count); in _wa_add()
168 kfree(wal->list); in _wa_add()
171 wal->list = list; in _wa_add()
175 unsigned int mid = start + (end - start) / 2; in _wa_add()
177 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { in _wa_add()
179 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { in _wa_add()
182 wa_ = &wal->list[mid]; in _wa_add()
184 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) { in _wa_add()
185 drm_err(&i915->drm, in _wa_add()
187 i915_mmio_reg_offset(wa_->reg), in _wa_add()
188 wa_->clr, wa_->set); in _wa_add()
190 wa_->set &= ~wa->clr; in _wa_add()
193 wal->wa_count++; in _wa_add()
194 wa_->set |= wa->set; in _wa_add()
195 wa_->clr |= wa->clr; in _wa_add()
196 wa_->read |= wa->read; in _wa_add()
201 wal->wa_count++; in _wa_add()
202 wa_ = &wal->list[wal->count++]; in _wa_add()
205 while (wa_-- > wal->list) { in _wa_add()
289 * documented as "masked" in b-spec. Its purpose is to allow writing to just a
336 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, in gen6_ctx_workarounds_init() argument
342 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine, in gen7_ctx_workarounds_init() argument
348 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, in gen8_ctx_workarounds_init() argument
360 /* Use Force Non-Coherent whenever executing a 3D context. This is a in gen8_ctx_workarounds_init()
371 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping in gen8_ctx_workarounds_init()
396 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine, in bdw_ctx_workarounds_init() argument
399 struct drm_i915_private *i915 = engine->i915; in bdw_ctx_workarounds_init()
401 gen8_ctx_workarounds_init(engine, wal); in bdw_ctx_workarounds_init()
403 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ in bdw_ctx_workarounds_init()
420 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ in bdw_ctx_workarounds_init()
421 (INTEL_INFO(i915)->gt == 3 ? HDC_FENCE_DEST_SLM_DISABLE : 0)); in bdw_ctx_workarounds_init()
424 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine, in chv_ctx_workarounds_init() argument
427 gen8_ctx_workarounds_init(engine, wal); in chv_ctx_workarounds_init()
436 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine, in gen9_ctx_workarounds_init() argument
439 struct drm_i915_private *i915 = engine->i915; in gen9_ctx_workarounds_init()
444 * Must match Display Engine. See in gen9_ctx_workarounds_init()
488 * Use Force Non-Coherent whenever executing a 3D context. This in gen9_ctx_workarounds_init()
509 * Supporting preemption with fine-granularity requires changes in the in gen9_ctx_workarounds_init()
512 * still able to use more fine-grained preemption levels, since in in gen9_ctx_workarounds_init()
514 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are in gen9_ctx_workarounds_init()
532 static void skl_tune_iz_hashing(struct intel_engine_cs *engine, in skl_tune_iz_hashing() argument
535 struct intel_gt *gt = engine->gt; in skl_tune_iz_hashing()
546 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i])) in skl_tune_iz_hashing()
553 * -> 0 <= ss <= 3; in skl_tune_iz_hashing()
555 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1; in skl_tune_iz_hashing()
556 vals[i] = 3 - ss; in skl_tune_iz_hashing()
572 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine, in skl_ctx_workarounds_init() argument
575 gen9_ctx_workarounds_init(engine, wal); in skl_ctx_workarounds_init()
576 skl_tune_iz_hashing(engine, wal); in skl_ctx_workarounds_init()
579 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine, in bxt_ctx_workarounds_init() argument
582 gen9_ctx_workarounds_init(engine, wal); in bxt_ctx_workarounds_init()
593 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, in kbl_ctx_workarounds_init() argument
596 struct drm_i915_private *i915 = engine->i915; in kbl_ctx_workarounds_init()
598 gen9_ctx_workarounds_init(engine, wal); in kbl_ctx_workarounds_init()
610 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine, in glk_ctx_workarounds_init() argument
613 gen9_ctx_workarounds_init(engine, wal); in glk_ctx_workarounds_init()
620 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, in cfl_ctx_workarounds_init() argument
623 gen9_ctx_workarounds_init(engine, wal); in cfl_ctx_workarounds_init()
634 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, in icl_ctx_workarounds_init() argument
652 0 /* write-only, so skip validation */, in icl_ctx_workarounds_init()
678 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine, in dg2_ctx_gt_tuning_init() argument
688 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, in gen12_ctx_workarounds_init() argument
691 struct drm_i915_private *i915 = engine->i915; in gen12_ctx_workarounds_init()
694 * Wa_1409142259:tgl,dg1,adl-p,adl-n in gen12_ctx_workarounds_init()
695 * Wa_1409347922:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
696 * Wa_1409252684:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
697 * Wa_1409217633:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
698 * Wa_1409207793:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
699 * Wa_1409178076:tgl,dg1,adl-p,adl-n in gen12_ctx_workarounds_init()
700 * Wa_1408979724:tgl,dg1,adl-p,adl-n in gen12_ctx_workarounds_init()
701 * Wa_14010443199:tgl,rkl,dg1,adl-p,adl-n in gen12_ctx_workarounds_init()
702 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p,adl-n in gen12_ctx_workarounds_init()
703 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p,adl-n in gen12_ctx_workarounds_init()
704 * Wa_22010465259:tgl,rkl,dg1,adl-s,adl-p,adl-n in gen12_ctx_workarounds_init()
715 * Wa_16011163337 - GS_TIMER in gen12_ctx_workarounds_init()
753 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, in dg1_ctx_workarounds_init() argument
756 gen12_ctx_workarounds_init(engine, wal); in dg1_ctx_workarounds_init()
767 static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine, in dg2_ctx_workarounds_init() argument
770 dg2_ctx_gt_tuning_init(engine, wal); in dg2_ctx_workarounds_init()
789 static void xelpg_ctx_gt_tuning_init(struct intel_engine_cs *engine, in xelpg_ctx_gt_tuning_init() argument
792 struct intel_gt *gt = engine->gt; in xelpg_ctx_gt_tuning_init()
794 dg2_ctx_gt_tuning_init(engine, wal); in xelpg_ctx_gt_tuning_init()
806 static void xelpg_ctx_workarounds_init(struct intel_engine_cs *engine, in xelpg_ctx_workarounds_init() argument
809 struct intel_gt *gt = engine->gt; in xelpg_ctx_workarounds_init()
811 xelpg_ctx_gt_tuning_init(engine, wal); in xelpg_ctx_workarounds_init()
837 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, in fakewa_disable_nestedbb_mode() argument
842 * maintain reliable, backward-compatible behavior for userspace with in fakewa_disable_nestedbb_mode()
845 * The per-context setting of MI_MODE[12] determines whether the bits in fakewa_disable_nestedbb_mode()
849 * into 3rd-level batchbuffers. When this new capability was first in fakewa_disable_nestedbb_mode()
855 * From a SW perspective, we want to maintain the backward-compatible in fakewa_disable_nestedbb_mode()
859 * userspace that utilizes third-level batchbuffers, so this will avoid in fakewa_disable_nestedbb_mode()
862 * consumers that want to utilize third-level batch nesting, we can in fakewa_disable_nestedbb_mode()
863 * provide a context parameter to allow them to opt-in. in fakewa_disable_nestedbb_mode()
865 wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); in fakewa_disable_nestedbb_mode()
868 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine, in gen12_ctx_gt_mocs_init() argument
876 * BLIT_CCTL registers are needed to be programmed to un-cached. in gen12_ctx_gt_mocs_init()
878 if (engine->class == COPY_ENGINE_CLASS) { in gen12_ctx_gt_mocs_init()
879 mocs = engine->gt->mocs.uc_index; in gen12_ctx_gt_mocs_init()
881 BLIT_CCTL(engine->mmio_base), in gen12_ctx_gt_mocs_init()
894 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine, in gen12_ctx_gt_fake_wa_init() argument
897 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in gen12_ctx_gt_fake_wa_init()
898 fakewa_disable_nestedbb_mode(engine, wal); in gen12_ctx_gt_fake_wa_init()
900 gen12_ctx_gt_mocs_init(engine, wal); in gen12_ctx_gt_fake_wa_init()
904 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, in __intel_engine_init_ctx_wa() argument
908 struct drm_i915_private *i915 = engine->i915; in __intel_engine_init_ctx_wa()
910 wa_init_start(wal, engine->gt, name, engine->name); in __intel_engine_init_ctx_wa()
918 gen12_ctx_gt_fake_wa_init(engine, wal); in __intel_engine_init_ctx_wa()
920 if (engine->class != RENDER_CLASS) in __intel_engine_init_ctx_wa()
923 if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74))) in __intel_engine_init_ctx_wa()
924 xelpg_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
926 dg2_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
928 dg1_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
930 gen12_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
932 icl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
934 cfl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
936 glk_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
938 kbl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
940 bxt_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
942 skl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
944 chv_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
946 bdw_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
948 gen7_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
950 gen6_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
960 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) in intel_engine_init_ctx_wa() argument
962 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context"); in intel_engine_init_ctx_wa()
967 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; in intel_engine_emit_ctx_wa()
968 struct intel_uncore *uncore = rq->engine->uncore; in intel_engine_emit_ctx_wa()
976 if (wal->count == 0) in intel_engine_emit_ctx_wa()
979 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); in intel_engine_emit_ctx_wa()
983 if ((IS_GFX_GT_IP_RANGE(rq->engine->gt, IP_VER(12, 70), IP_VER(12, 74)) || in intel_engine_emit_ctx_wa()
984 IS_DG2(rq->i915)) && rq->engine->class == RENDER_CLASS) in intel_engine_emit_ctx_wa()
985 cs = intel_ring_begin(rq, (wal->count * 2 + 6)); in intel_engine_emit_ctx_wa()
987 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); in intel_engine_emit_ctx_wa()
994 intel_gt_mcr_lock(wal->gt, &flags); in intel_engine_emit_ctx_wa()
995 spin_lock(&uncore->lock); in intel_engine_emit_ctx_wa()
998 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); in intel_engine_emit_ctx_wa()
999 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in intel_engine_emit_ctx_wa()
1003 if (wa->masked_reg || (wa->clr | wa->set) == U32_MAX) { in intel_engine_emit_ctx_wa()
1004 val = wa->set; in intel_engine_emit_ctx_wa()
1006 val = wa->is_mcr ? in intel_engine_emit_ctx_wa()
1007 intel_gt_mcr_read_any_fw(wal->gt, wa->mcr_reg) : in intel_engine_emit_ctx_wa()
1008 intel_uncore_read_fw(uncore, wa->reg); in intel_engine_emit_ctx_wa()
1009 val &= ~wa->clr; in intel_engine_emit_ctx_wa()
1010 val |= wa->set; in intel_engine_emit_ctx_wa()
1013 *cs++ = i915_mmio_reg_offset(wa->reg); in intel_engine_emit_ctx_wa()
1019 if ((IS_GFX_GT_IP_RANGE(rq->engine->gt, IP_VER(12, 70), IP_VER(12, 74)) || in intel_engine_emit_ctx_wa()
1020 IS_DG2(rq->i915)) && rq->engine->class == RENDER_CLASS) { in intel_engine_emit_ctx_wa()
1028 spin_unlock(&uncore->lock); in intel_engine_emit_ctx_wa()
1029 intel_gt_mcr_unlock(wal->gt, flags); in intel_engine_emit_ctx_wa()
1033 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); in intel_engine_emit_ctx_wa()
1102 /* L3 caching of data atomics doesn't work -- disable it. */ in hsw_gt_workarounds_init()
1117 const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu; in gen9_wa_init_mcr()
1125 * Before any MMIO read into slice/subslice specific registers, MCR in gen9_wa_init_mcr()
1129 * specific s/ss combination, but this is OK since these registers in gen9_wa_init_mcr()
1134 slice = ffs(sseu->slice_mask) - 1; in gen9_wa_init_mcr()
1135 GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw)); in gen9_wa_init_mcr()
1138 subslice--; in gen9_wa_init_mcr()
1147 drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr); in gen9_wa_init_mcr()
1155 struct drm_i915_private *i915 = gt->i915; in gen9_gt_workarounds_init()
1169 * Must match Display Engine. See in gen9_gt_workarounds_init()
1194 if (IS_SKYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0)) in skl_gt_workarounds_init()
1206 if (IS_KABYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, 0, STEP_C0)) in kbl_gt_workarounds_init()
1258 struct drm_printer p = drm_dbg_printer(>->i915->drm, DRM_UT_DRIVER, in debug_dump_steering()
1270 gt->default_steering.groupid = slice; in __add_mcr_wa()
1271 gt->default_steering.instanceid = subslice; in __add_mcr_wa()
1279 const struct sseu_dev_info *sseu = >->info.sseu; in icl_wa_init_mcr()
1282 GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11); in icl_wa_init_mcr()
1283 GEM_BUG_ON(hweight8(sseu->slice_mask) > 1); in icl_wa_init_mcr()
1299 * worry about explicitly re-steering L3BANK reads later. in icl_wa_init_mcr()
1301 if (gt->info.l3bank_mask & BIT(subslice)) in icl_wa_init_mcr()
1302 gt->steering_table[L3BANK] = NULL; in icl_wa_init_mcr()
1310 const struct sseu_dev_info *sseu = >->info.sseu; in xehp_init_mcr()
1319 * - GSLICE (fusable) in xehp_init_mcr()
1320 * - DSS (sub-unit within gslice; fusable) in xehp_init_mcr()
1321 * - L3 Bank (fusable) in xehp_init_mcr()
1322 * - MSLICE (fusable) in xehp_init_mcr()
1323 * - LNCF (sub-unit within mslice; always present if mslice is present) in xehp_init_mcr()
1342 slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask, in xehp_init_mcr()
1349 for_each_set_bit(i, >->info.mslice_mask, GEN12_MAX_MSLICES) in xehp_init_mcr()
1358 gt->steering_table[LNCF] = NULL; in xehp_init_mcr()
1362 if (slice_mask & gt->info.mslice_mask) { in xehp_init_mcr()
1363 slice_mask &= gt->info.mslice_mask; in xehp_init_mcr()
1364 gt->steering_table[MSLICE] = NULL; in xehp_init_mcr()
1376 * DG2-G10, any value in the steering registers will work fine since in xehp_init_mcr()
1377 * all instances are present, but DG2-G11 only has SQIDI instances at in xehp_init_mcr()
1389 if (IS_DG2(gt->i915)) in xehp_init_mcr()
1396 struct drm_i915_private *i915 = gt->i915; in icl_gt_workarounds_init()
1460 * Though there are per-engine instances of these registers,
1461 * they retain their value through engine resets and should
1463 * the engine-specific workaround list.
1468 struct intel_engine_cs *engine; in wa_14011060649() local
1471 for_each_engine(engine, gt, id) { in wa_14011060649()
1472 if (engine->class != VIDEO_DECODE_CLASS || in wa_14011060649()
1473 (engine->instance % 2)) in wa_14011060649()
1476 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), in wa_14011060649()
1486 /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */ in gen12_gt_workarounds_init()
1489 /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */ in gen12_gt_workarounds_init()
1515 /* Empirical testing shows this register is unaffected by engine reset. */ in dg1_gt_workarounds_init()
1527 if (IS_DG2_G10(gt->i915)) { in dg2_gt_workarounds_init()
1590 struct intel_engine_cs *engine; in wa_16021867713() local
1593 for_each_engine(engine, gt, id) in wa_16021867713()
1594 if (engine->class == VIDEO_DECODE_CLASS) in wa_16021867713()
1595 wa_write_or(wal, VDBOX_CGCTL3F1C(engine->mmio_base), in wa_16021867713()
1616 * Issue is seen on media KPI test running on VDBOX engine in xelpmp_gt_workarounds_init()
1634 * engine resets and also are not part of any engine's register state context.
1635 * I.e., settings that only need to be re-applied in the event of a full GT
1645 if (IS_DG2(gt->i915)) { in gt_tuning_settings()
1654 struct drm_i915_private *i915 = gt->i915; in gt_init_workarounds()
1658 if (gt->type == GT_MEDIA) { in gt_init_workarounds()
1709 struct i915_wa_list *wal = >->wa_list; in intel_gt_init_workarounds()
1720 if ((cur ^ wa->set) & wa->read) { in wa_verify()
1723 name, from, i915_mmio_reg_offset(wa->reg), in wa_verify()
1724 cur, cur & wa->read, wa->set & wa->read); in wa_verify()
1734 struct intel_gt *gt = wal->gt; in wa_list_apply()
1735 struct intel_uncore *uncore = gt->uncore; in wa_list_apply()
1741 if (!wal->count) in wa_list_apply()
1747 spin_lock(&uncore->lock); in wa_list_apply()
1750 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_apply()
1753 /* open-coded rmw due to steering */ in wa_list_apply()
1754 if (wa->clr) in wa_list_apply()
1755 old = wa->is_mcr ? in wa_list_apply()
1756 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_apply()
1757 intel_uncore_read_fw(uncore, wa->reg); in wa_list_apply()
1758 val = (old & ~wa->clr) | wa->set; in wa_list_apply()
1759 if (val != old || !wa->clr) { in wa_list_apply()
1760 if (wa->is_mcr) in wa_list_apply()
1761 intel_gt_mcr_multicast_write_fw(gt, wa->mcr_reg, val); in wa_list_apply()
1763 intel_uncore_write_fw(uncore, wa->reg, val); in wa_list_apply()
1767 u32 val = wa->is_mcr ? in wa_list_apply()
1768 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_apply()
1769 intel_uncore_read_fw(uncore, wa->reg); in wa_list_apply()
1771 wa_verify(gt, wa, val, wal->name, "application"); in wa_list_apply()
1776 spin_unlock(&uncore->lock); in wa_list_apply()
1782 wa_list_apply(>->wa_list); in intel_gt_apply_workarounds()
1789 struct intel_uncore *uncore = gt->uncore; in wa_list_verify()
1799 spin_lock(&uncore->lock); in wa_list_verify()
1802 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wa_list_verify()
1803 ok &= wa_verify(wal->gt, wa, wa->is_mcr ? in wa_list_verify()
1804 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_verify()
1805 intel_uncore_read_fw(uncore, wa->reg), in wa_list_verify()
1806 wal->name, from); in wa_list_verify()
1809 spin_unlock(&uncore->lock); in wa_list_verify()
1817 return wa_list_verify(gt, >->wa_list, from); in intel_gt_verify_workarounds()
1842 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_reg_ext()
1860 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_mcr_reg_ext()
1897 static void skl_whitelist_build(struct intel_engine_cs *engine) in skl_whitelist_build() argument
1899 struct i915_wa_list *w = &engine->whitelist; in skl_whitelist_build()
1901 if (engine->class != RENDER_CLASS) in skl_whitelist_build()
1910 static void bxt_whitelist_build(struct intel_engine_cs *engine) in bxt_whitelist_build() argument
1912 if (engine->class != RENDER_CLASS) in bxt_whitelist_build()
1915 gen9_whitelist_build(&engine->whitelist); in bxt_whitelist_build()
1918 static void kbl_whitelist_build(struct intel_engine_cs *engine) in kbl_whitelist_build() argument
1920 struct i915_wa_list *w = &engine->whitelist; in kbl_whitelist_build()
1922 if (engine->class != RENDER_CLASS) in kbl_whitelist_build()
1931 static void glk_whitelist_build(struct intel_engine_cs *engine) in glk_whitelist_build() argument
1933 struct i915_wa_list *w = &engine->whitelist; in glk_whitelist_build()
1935 if (engine->class != RENDER_CLASS) in glk_whitelist_build()
1944 static void cfl_whitelist_build(struct intel_engine_cs *engine) in cfl_whitelist_build() argument
1946 struct i915_wa_list *w = &engine->whitelist; in cfl_whitelist_build()
1948 if (engine->class != RENDER_CLASS) in cfl_whitelist_build()
1957 * - PS_INVOCATION_COUNT in cfl_whitelist_build()
1958 * - PS_INVOCATION_COUNT_UDW in cfl_whitelist_build()
1959 * - PS_DEPTH_COUNT in cfl_whitelist_build()
1960 * - PS_DEPTH_COUNT_UDW in cfl_whitelist_build()
1967 static void allow_read_ctx_timestamp(struct intel_engine_cs *engine) in allow_read_ctx_timestamp() argument
1969 struct i915_wa_list *w = &engine->whitelist; in allow_read_ctx_timestamp()
1971 if (engine->class != RENDER_CLASS) in allow_read_ctx_timestamp()
1973 RING_CTX_TIMESTAMP(engine->mmio_base), in allow_read_ctx_timestamp()
1977 static void cml_whitelist_build(struct intel_engine_cs *engine) in cml_whitelist_build() argument
1979 allow_read_ctx_timestamp(engine); in cml_whitelist_build()
1981 cfl_whitelist_build(engine); in cml_whitelist_build()
1984 static void icl_whitelist_build(struct intel_engine_cs *engine) in icl_whitelist_build() argument
1986 struct i915_wa_list *w = &engine->whitelist; in icl_whitelist_build()
1988 allow_read_ctx_timestamp(engine); in icl_whitelist_build()
1990 switch (engine->class) { in icl_whitelist_build()
2005 * - PS_INVOCATION_COUNT in icl_whitelist_build()
2006 * - PS_INVOCATION_COUNT_UDW in icl_whitelist_build()
2007 * - PS_DEPTH_COUNT in icl_whitelist_build()
2008 * - PS_DEPTH_COUNT_UDW in icl_whitelist_build()
2017 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base), in icl_whitelist_build()
2020 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base), in icl_whitelist_build()
2023 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base), in icl_whitelist_build()
2032 static void tgl_whitelist_build(struct intel_engine_cs *engine) in tgl_whitelist_build() argument
2034 struct i915_wa_list *w = &engine->whitelist; in tgl_whitelist_build()
2036 allow_read_ctx_timestamp(engine); in tgl_whitelist_build()
2038 switch (engine->class) { in tgl_whitelist_build()
2045 * - PS_INVOCATION_COUNT in tgl_whitelist_build()
2046 * - PS_INVOCATION_COUNT_UDW in tgl_whitelist_build()
2047 * - PS_DEPTH_COUNT in tgl_whitelist_build()
2048 * - PS_DEPTH_COUNT_UDW in tgl_whitelist_build()
2057 * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p in tgl_whitelist_build()
2073 static void dg2_whitelist_build(struct intel_engine_cs *engine) in dg2_whitelist_build() argument
2075 struct i915_wa_list *w = &engine->whitelist; in dg2_whitelist_build()
2077 switch (engine->class) { in dg2_whitelist_build()
2088 static void xelpg_whitelist_build(struct intel_engine_cs *engine) in xelpg_whitelist_build() argument
2090 struct i915_wa_list *w = &engine->whitelist; in xelpg_whitelist_build()
2092 switch (engine->class) { in xelpg_whitelist_build()
2103 void intel_engine_init_whitelist(struct intel_engine_cs *engine) in intel_engine_init_whitelist() argument
2105 struct drm_i915_private *i915 = engine->i915; in intel_engine_init_whitelist()
2106 struct i915_wa_list *w = &engine->whitelist; in intel_engine_init_whitelist()
2108 wa_init_start(w, engine->gt, "whitelist", engine->name); in intel_engine_init_whitelist()
2110 if (engine->gt->type == GT_MEDIA) in intel_engine_init_whitelist()
2112 else if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74))) in intel_engine_init_whitelist()
2113 xelpg_whitelist_build(engine); in intel_engine_init_whitelist()
2115 dg2_whitelist_build(engine); in intel_engine_init_whitelist()
2117 tgl_whitelist_build(engine); in intel_engine_init_whitelist()
2119 icl_whitelist_build(engine); in intel_engine_init_whitelist()
2121 cml_whitelist_build(engine); in intel_engine_init_whitelist()
2123 cfl_whitelist_build(engine); in intel_engine_init_whitelist()
2125 glk_whitelist_build(engine); in intel_engine_init_whitelist()
2127 kbl_whitelist_build(engine); in intel_engine_init_whitelist()
2129 bxt_whitelist_build(engine); in intel_engine_init_whitelist()
2131 skl_whitelist_build(engine); in intel_engine_init_whitelist()
2140 void intel_engine_apply_whitelist(struct intel_engine_cs *engine) in intel_engine_apply_whitelist() argument
2142 const struct i915_wa_list *wal = &engine->whitelist; in intel_engine_apply_whitelist()
2143 struct intel_uncore *uncore = engine->uncore; in intel_engine_apply_whitelist()
2144 const u32 base = engine->mmio_base; in intel_engine_apply_whitelist()
2148 if (!wal->count) in intel_engine_apply_whitelist()
2151 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in intel_engine_apply_whitelist()
2154 i915_mmio_reg_offset(wa->reg)); in intel_engine_apply_whitelist()
2171 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_fake_wa_init() argument
2181 * spec in certain circumstances on specific platforms. in engine_fake_wa_init()
2183 if (GRAPHICS_VER(engine->i915) >= 12) { in engine_fake_wa_init()
2184 mocs_r = engine->gt->mocs.uc_index; in engine_fake_wa_init()
2185 mocs_w = engine->gt->mocs.uc_index; in engine_fake_wa_init()
2187 if (HAS_L3_CCS_READ(engine->i915) && in engine_fake_wa_init()
2188 engine->class == COMPUTE_CLASS) { in engine_fake_wa_init()
2189 mocs_r = engine->gt->mocs.wb_index; in engine_fake_wa_init()
2197 drm_WARN_ON(&engine->i915->drm, mocs_r == 0); in engine_fake_wa_init()
2201 RING_CMD_CCTL(engine->mmio_base), in engine_fake_wa_init()
2208 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in rcs_engine_wa_init() argument
2210 struct drm_i915_private *i915 = engine->i915; in rcs_engine_wa_init()
2211 struct intel_gt *gt = engine->gt; in rcs_engine_wa_init()
2255 * Wa_1606700617:tgl,dg1,adl-p in rcs_engine_wa_init()
2256 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p in rcs_engine_wa_init()
2257 * Wa_14010826681:tgl,dg1,rkl,adl-p in rcs_engine_wa_init()
2267 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */ in rcs_engine_wa_init()
2274 * Wa_14010919138:rkl,dg1,adl-s,adl-p in rcs_engine_wa_init()
2279 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */ in rcs_engine_wa_init()
2300 * BSpec; some indicate this is an A0-only WA, others indicate in rcs_engine_wa_init()
2383 * Intel platforms that support fine-grained preemption (i.e., gen9 and in rcs_engine_wa_init()
2384 * beyond) allow the kernel-mode driver to choose between two different in rcs_engine_wa_init()
2389 * kernel-only register CS_DEBUG_MODE1 (0x20EC). Any granularity in rcs_engine_wa_init()
2390 * and settings chosen by the kernel-mode driver will apply to all in rcs_engine_wa_init()
2394 * Preemption settings are controlled on a per-context basis via in rcs_engine_wa_init()
2402 * that name is somewhat misleading as other non-granularity in rcs_engine_wa_init()
2408 * userspace developed before object-level preemption was enabled would in rcs_engine_wa_init()
2411 * object-level preemption disabled by default (see in rcs_engine_wa_init()
2413 * userspace drivers could opt-in to object-level preemption as they in rcs_engine_wa_init()
2414 * saw fit. For post-gen9 platforms, we continue to utilize Option 2; in rcs_engine_wa_init()
2421 * - Wa_14015141709: On DG2 and early steppings of MTL, in rcs_engine_wa_init()
2422 * CS_CHICKEN1[0] does not disable object-level preemption as in rcs_engine_wa_init()
2425 * to disable object-level preemption on these platforms/steppings in rcs_engine_wa_init()
2428 * - Wa_16013994831: May require that userspace program in rcs_engine_wa_init()
2556 if (INTEL_INFO(i915)->gt == 1) in rcs_engine_wa_init()
2685 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in xcs_engine_wa_init() argument
2687 struct drm_i915_private *i915 = engine->i915; in xcs_engine_wa_init()
2692 RING_SEMA_WAIT_POLL(engine->mmio_base), in xcs_engine_wa_init()
2696 if (NEEDS_FASTCOLOR_BLT_WABB(engine)) in xcs_engine_wa_init()
2697 wa_masked_field_set(wal, ECOSKPD(engine->mmio_base), in xcs_engine_wa_init()
2703 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in ccs_engine_wa_init() argument
2705 /* boilerplate for any CCS engine workaround */ in ccs_engine_wa_init()
2712 * the GuC save/restore lists, re-applied at the right times, and checked for
2716 * part of an engine's register state context. If a register is part of a
2724 struct drm_i915_private *i915 = gt->i915; in add_render_compute_tuning_settings()
2730 * This tuning setting proves beneficial only on ATS-M designs; the in add_render_compute_tuning_settings()
2734 if (INTEL_INFO(i915)->tuning_thread_rr_after_dep) in add_render_compute_tuning_settings()
2742 static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal) in ccs_engine_wa_mode() argument
2744 struct intel_gt *gt = engine->gt; in ccs_engine_wa_mode()
2747 if (!IS_DG2(gt->i915)) in ccs_engine_wa_mode()
2769 * specific engine. Since all render+compute engines get reset
2772 * here and then add them to just a single RCS or CCS engine's
2773 * workaround list (whichever engine has the XXXX flag).
2776 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in general_render_compute_wa_init() argument
2778 struct drm_i915_private *i915 = engine->i915; in general_render_compute_wa_init()
2779 struct intel_gt *gt = engine->gt; in general_render_compute_wa_init()
2874 * Note that register 0xE420 is write-only and cannot be read in general_render_compute_wa_init()
2880 0 /* write-only, so skip validation */, in general_render_compute_wa_init()
2886 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_init_workarounds() argument
2888 if (GRAPHICS_VER(engine->i915) < 4) in engine_init_workarounds()
2891 engine_fake_wa_init(engine, wal); in engine_init_workarounds()
2895 * to a single RCS/CCS engine's workaround list since in engine_init_workarounds()
2898 if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) { in engine_init_workarounds()
2899 general_render_compute_wa_init(engine, wal); in engine_init_workarounds()
2900 ccs_engine_wa_mode(engine, wal); in engine_init_workarounds()
2903 if (engine->class == COMPUTE_CLASS) in engine_init_workarounds()
2904 ccs_engine_wa_init(engine, wal); in engine_init_workarounds()
2905 else if (engine->class == RENDER_CLASS) in engine_init_workarounds()
2906 rcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2908 xcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2911 void intel_engine_init_workarounds(struct intel_engine_cs *engine) in intel_engine_init_workarounds() argument
2913 struct i915_wa_list *wal = &engine->wa_list; in intel_engine_init_workarounds()
2915 wa_init_start(wal, engine->gt, "engine", engine->name); in intel_engine_init_workarounds()
2916 engine_init_workarounds(engine, wal); in intel_engine_init_workarounds()
2920 void intel_engine_apply_workarounds(struct intel_engine_cs *engine) in intel_engine_apply_workarounds() argument
2922 wa_list_apply(&engine->wa_list); in intel_engine_apply_workarounds()
2992 struct drm_i915_private *i915 = rq->i915; in wa_list_srm()
3001 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
3002 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg))) in wa_list_srm()
3010 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
3011 u32 offset = i915_mmio_reg_offset(wa->reg); in wa_list_srm()
3038 if (!wal->count) in engine_wa_list_verify()
3041 vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm, in engine_wa_list_verify()
3042 wal->count * sizeof(u32)); in engine_wa_list_verify()
3046 intel_engine_pm_get(ce->engine); in engine_wa_list_verify()
3049 err = i915_gem_object_lock(vma->obj, &ww); in engine_wa_list_verify()
3079 err = -ETIME; in engine_wa_list_verify()
3083 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); in engine_wa_list_verify()
3090 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in engine_wa_list_verify()
3091 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg))) in engine_wa_list_verify()
3094 if (!wa_verify(wal->gt, wa, results[i], wal->name, from)) in engine_wa_list_verify()
3095 err = -ENXIO; in engine_wa_list_verify()
3098 i915_gem_object_unpin_map(vma->obj); in engine_wa_list_verify()
3107 if (err == -EDEADLK) { in engine_wa_list_verify()
3113 intel_engine_pm_put(ce->engine); in engine_wa_list_verify()
3118 int intel_engine_verify_workarounds(struct intel_engine_cs *engine, in intel_engine_verify_workarounds() argument
3121 return engine_wa_list_verify(engine->kernel_context, in intel_engine_verify_workarounds()
3122 &engine->wa_list, in intel_engine_verify_workarounds()