Lines Matching +full:engine +full:- +full:specific
1 // SPDX-License-Identifier: MIT
3 * Copyright © 2014-2018 Intel Corporation
27 * - Context workarounds: workarounds that touch registers that are
37 * - Engine workarounds: the list of these WAs is applied whenever the specific
38 * engine is reset. It's also possible that a set of engine classes share a
42 * driver is to tie those workarounds to the first compute/render engine that
43 * is registered. When executing with GuC submission, engine resets are
45 * written once, on engine initialization, and then passed to GuC, that
49 * Workarounds for registers specific to RCS and CCS should be implemented in
52 * xcs_engine_wa_init(). Workarounds for registers not belonging to a specific
53 * engine's MMIO range but that are part of of the common RCS/CCS reset domain
56 * - GT workarounds: the list of these WAs is applied whenever these registers
62 * - Register whitelist: some workarounds need to be implemented in userspace,
66 * these to/be-whitelisted registers to some special HW registers).
71 * - Workaround batchbuffers: buffers that get executed automatically by the
84 * engine registers are restored in a context restore sequence. This is
87 * - Other: There are WAs that, due to their nature, cannot be applied from a
99 wal->gt = gt; in wa_init_start()
100 wal->name = name; in wa_init_start()
101 wal->engine_name = engine_name; in wa_init_start()
109 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { in wa_init_finish()
110 struct i915_wa *list = kmemdup(wal->list, in wa_init_finish()
111 wal->count * sizeof(*list), in wa_init_finish()
115 kfree(wal->list); in wa_init_finish()
116 wal->list = list; in wa_init_finish()
120 if (!wal->count) in wa_init_finish()
123 gt_dbg(wal->gt, "Initialized %u %s workarounds on %s\n", in wa_init_finish()
124 wal->wa_count, wal->name, wal->engine_name); in wa_init_finish()
134 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wal_get_fw_for_rmw()
136 wa->reg, in wal_get_fw_for_rmw()
145 unsigned int addr = i915_mmio_reg_offset(wa->reg); in _wa_add()
146 struct drm_i915_private *i915 = wal->gt->i915; in _wa_add()
147 unsigned int start = 0, end = wal->count; in _wa_add()
153 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ in _wa_add()
156 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), in _wa_add()
159 drm_err(&i915->drm, "No space for workaround init!\n"); in _wa_add()
163 if (wal->list) { in _wa_add()
164 memcpy(list, wal->list, sizeof(*wa) * wal->count); in _wa_add()
165 kfree(wal->list); in _wa_add()
168 wal->list = list; in _wa_add()
172 unsigned int mid = start + (end - start) / 2; in _wa_add()
174 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { in _wa_add()
176 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { in _wa_add()
179 wa_ = &wal->list[mid]; in _wa_add()
181 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) { in _wa_add()
182 drm_err(&i915->drm, in _wa_add()
184 i915_mmio_reg_offset(wa_->reg), in _wa_add()
185 wa_->clr, wa_->set); in _wa_add()
187 wa_->set &= ~wa->clr; in _wa_add()
190 wal->wa_count++; in _wa_add()
191 wa_->set |= wa->set; in _wa_add()
192 wa_->clr |= wa->clr; in _wa_add()
193 wa_->read |= wa->read; in _wa_add()
198 wal->wa_count++; in _wa_add()
199 wa_ = &wal->list[wal->count++]; in _wa_add()
202 while (wa_-- > wal->list) { in _wa_add()
292 * documented as "masked" in b-spec. Its purpose is to allow writing to just a
339 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, in gen6_ctx_workarounds_init() argument
345 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine, in gen7_ctx_workarounds_init() argument
351 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, in gen8_ctx_workarounds_init() argument
363 /* Use Force Non-Coherent whenever executing a 3D context. This is a in gen8_ctx_workarounds_init()
374 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping in gen8_ctx_workarounds_init()
399 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine, in bdw_ctx_workarounds_init() argument
402 struct drm_i915_private *i915 = engine->i915; in bdw_ctx_workarounds_init()
404 gen8_ctx_workarounds_init(engine, wal); in bdw_ctx_workarounds_init()
406 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ in bdw_ctx_workarounds_init()
423 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ in bdw_ctx_workarounds_init()
427 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine, in chv_ctx_workarounds_init() argument
430 gen8_ctx_workarounds_init(engine, wal); in chv_ctx_workarounds_init()
439 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine, in gen9_ctx_workarounds_init() argument
442 struct drm_i915_private *i915 = engine->i915; in gen9_ctx_workarounds_init()
447 * Must match Display Engine. See in gen9_ctx_workarounds_init()
491 * Use Force Non-Coherent whenever executing a 3D context. This in gen9_ctx_workarounds_init()
512 * Supporting preemption with fine-granularity requires changes in the in gen9_ctx_workarounds_init()
515 * still able to use more fine-grained preemption levels, since in in gen9_ctx_workarounds_init()
517 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are in gen9_ctx_workarounds_init()
535 static void skl_tune_iz_hashing(struct intel_engine_cs *engine, in skl_tune_iz_hashing() argument
538 struct intel_gt *gt = engine->gt; in skl_tune_iz_hashing()
549 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i])) in skl_tune_iz_hashing()
556 * -> 0 <= ss <= 3; in skl_tune_iz_hashing()
558 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1; in skl_tune_iz_hashing()
559 vals[i] = 3 - ss; in skl_tune_iz_hashing()
575 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine, in skl_ctx_workarounds_init() argument
578 gen9_ctx_workarounds_init(engine, wal); in skl_ctx_workarounds_init()
579 skl_tune_iz_hashing(engine, wal); in skl_ctx_workarounds_init()
582 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine, in bxt_ctx_workarounds_init() argument
585 gen9_ctx_workarounds_init(engine, wal); in bxt_ctx_workarounds_init()
596 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, in kbl_ctx_workarounds_init() argument
599 struct drm_i915_private *i915 = engine->i915; in kbl_ctx_workarounds_init()
601 gen9_ctx_workarounds_init(engine, wal); in kbl_ctx_workarounds_init()
613 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine, in glk_ctx_workarounds_init() argument
616 gen9_ctx_workarounds_init(engine, wal); in glk_ctx_workarounds_init()
623 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, in cfl_ctx_workarounds_init() argument
626 gen9_ctx_workarounds_init(engine, wal); in cfl_ctx_workarounds_init()
637 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, in icl_ctx_workarounds_init() argument
655 0 /* write-only, so skip validation */, in icl_ctx_workarounds_init()
681 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine, in dg2_ctx_gt_tuning_init() argument
691 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, in gen12_ctx_workarounds_init() argument
694 struct drm_i915_private *i915 = engine->i915; in gen12_ctx_workarounds_init()
697 * Wa_1409142259:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
698 * Wa_1409347922:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
699 * Wa_1409252684:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
700 * Wa_1409217633:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
701 * Wa_1409207793:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
702 * Wa_1409178076:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
703 * Wa_1408979724:tgl,dg1,adl-p in gen12_ctx_workarounds_init()
704 * Wa_14010443199:tgl,rkl,dg1,adl-p in gen12_ctx_workarounds_init()
705 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p in gen12_ctx_workarounds_init()
706 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p in gen12_ctx_workarounds_init()
717 * Wa_16011163337 - GS_TIMER in gen12_ctx_workarounds_init()
749 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, in dg1_ctx_workarounds_init() argument
752 gen12_ctx_workarounds_init(engine, wal); in dg1_ctx_workarounds_init()
763 static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine, in dg2_ctx_workarounds_init() argument
766 dg2_ctx_gt_tuning_init(engine, wal); in dg2_ctx_workarounds_init()
785 static void xelpg_ctx_gt_tuning_init(struct intel_engine_cs *engine, in xelpg_ctx_gt_tuning_init() argument
788 struct intel_gt *gt = engine->gt; in xelpg_ctx_gt_tuning_init()
790 dg2_ctx_gt_tuning_init(engine, wal); in xelpg_ctx_gt_tuning_init()
797 static void xelpg_ctx_workarounds_init(struct intel_engine_cs *engine, in xelpg_ctx_workarounds_init() argument
800 struct intel_gt *gt = engine->gt; in xelpg_ctx_workarounds_init()
802 xelpg_ctx_gt_tuning_init(engine, wal); in xelpg_ctx_workarounds_init()
825 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, in fakewa_disable_nestedbb_mode() argument
830 * maintain reliable, backward-compatible behavior for userspace with in fakewa_disable_nestedbb_mode()
833 * The per-context setting of MI_MODE[12] determines whether the bits in fakewa_disable_nestedbb_mode()
837 * into 3rd-level batchbuffers. When this new capability was first in fakewa_disable_nestedbb_mode()
843 * From a SW perspective, we want to maintain the backward-compatible in fakewa_disable_nestedbb_mode()
847 * userspace that utilizes third-level batchbuffers, so this will avoid in fakewa_disable_nestedbb_mode()
850 * consumers that want to utilize third-level batch nesting, we can in fakewa_disable_nestedbb_mode()
851 * provide a context parameter to allow them to opt-in. in fakewa_disable_nestedbb_mode()
853 wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); in fakewa_disable_nestedbb_mode()
856 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine, in gen12_ctx_gt_mocs_init() argument
864 * BLIT_CCTL registers are needed to be programmed to un-cached. in gen12_ctx_gt_mocs_init()
866 if (engine->class == COPY_ENGINE_CLASS) { in gen12_ctx_gt_mocs_init()
867 mocs = engine->gt->mocs.uc_index; in gen12_ctx_gt_mocs_init()
869 BLIT_CCTL(engine->mmio_base), in gen12_ctx_gt_mocs_init()
882 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine, in gen12_ctx_gt_fake_wa_init() argument
885 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in gen12_ctx_gt_fake_wa_init()
886 fakewa_disable_nestedbb_mode(engine, wal); in gen12_ctx_gt_fake_wa_init()
888 gen12_ctx_gt_mocs_init(engine, wal); in gen12_ctx_gt_fake_wa_init()
892 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, in __intel_engine_init_ctx_wa() argument
896 struct drm_i915_private *i915 = engine->i915; in __intel_engine_init_ctx_wa()
898 wa_init_start(wal, engine->gt, name, engine->name); in __intel_engine_init_ctx_wa()
906 gen12_ctx_gt_fake_wa_init(engine, wal); in __intel_engine_init_ctx_wa()
908 if (engine->class != RENDER_CLASS) in __intel_engine_init_ctx_wa()
911 if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 71))) in __intel_engine_init_ctx_wa()
912 xelpg_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
916 dg2_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
920 dg1_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
922 gen12_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
924 icl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
926 cfl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
928 glk_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
930 kbl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
932 bxt_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
934 skl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
936 chv_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
938 bdw_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
940 gen7_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
942 gen6_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
952 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) in intel_engine_init_ctx_wa() argument
954 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context"); in intel_engine_init_ctx_wa()
959 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; in intel_engine_emit_ctx_wa()
960 struct intel_uncore *uncore = rq->engine->uncore; in intel_engine_emit_ctx_wa()
968 if (wal->count == 0) in intel_engine_emit_ctx_wa()
971 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); in intel_engine_emit_ctx_wa()
975 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); in intel_engine_emit_ctx_wa()
981 intel_gt_mcr_lock(wal->gt, &flags); in intel_engine_emit_ctx_wa()
982 spin_lock(&uncore->lock); in intel_engine_emit_ctx_wa()
985 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); in intel_engine_emit_ctx_wa()
986 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in intel_engine_emit_ctx_wa()
990 if (wa->masked_reg || (wa->clr | wa->set) == U32_MAX) { in intel_engine_emit_ctx_wa()
991 val = wa->set; in intel_engine_emit_ctx_wa()
993 val = wa->is_mcr ? in intel_engine_emit_ctx_wa()
994 intel_gt_mcr_read_any_fw(wal->gt, wa->mcr_reg) : in intel_engine_emit_ctx_wa()
995 intel_uncore_read_fw(uncore, wa->reg); in intel_engine_emit_ctx_wa()
996 val &= ~wa->clr; in intel_engine_emit_ctx_wa()
997 val |= wa->set; in intel_engine_emit_ctx_wa()
1000 *cs++ = i915_mmio_reg_offset(wa->reg); in intel_engine_emit_ctx_wa()
1006 spin_unlock(&uncore->lock); in intel_engine_emit_ctx_wa()
1007 intel_gt_mcr_unlock(wal->gt, flags); in intel_engine_emit_ctx_wa()
1011 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); in intel_engine_emit_ctx_wa()
1080 /* L3 caching of data atomics doesn't work -- disable it. */ in hsw_gt_workarounds_init()
1095 const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu; in gen9_wa_init_mcr()
1103 * Before any MMIO read into slice/subslice specific registers, MCR in gen9_wa_init_mcr()
1107 * specific s/ss combination, but this is OK since these registers in gen9_wa_init_mcr()
1112 slice = ffs(sseu->slice_mask) - 1; in gen9_wa_init_mcr()
1113 GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw)); in gen9_wa_init_mcr()
1116 subslice--; in gen9_wa_init_mcr()
1125 drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr); in gen9_wa_init_mcr()
1133 struct drm_i915_private *i915 = gt->i915; in gen9_gt_workarounds_init()
1147 * Must match Display Engine. See in gen9_gt_workarounds_init()
1172 if (IS_SKYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0)) in skl_gt_workarounds_init()
1184 if (IS_KABYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, 0, STEP_C0)) in kbl_gt_workarounds_init()
1247 gt->default_steering.groupid = slice; in __add_mcr_wa()
1248 gt->default_steering.instanceid = subslice; in __add_mcr_wa()
1256 const struct sseu_dev_info *sseu = >->info.sseu; in icl_wa_init_mcr()
1259 GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11); in icl_wa_init_mcr()
1260 GEM_BUG_ON(hweight8(sseu->slice_mask) > 1); in icl_wa_init_mcr()
1276 * worry about explicitly re-steering L3BANK reads later. in icl_wa_init_mcr()
1278 if (gt->info.l3bank_mask & BIT(subslice)) in icl_wa_init_mcr()
1279 gt->steering_table[L3BANK] = NULL; in icl_wa_init_mcr()
1287 const struct sseu_dev_info *sseu = >->info.sseu; in xehp_init_mcr()
1296 * - GSLICE (fusable) in xehp_init_mcr()
1297 * - DSS (sub-unit within gslice; fusable) in xehp_init_mcr()
1298 * - L3 Bank (fusable) in xehp_init_mcr()
1299 * - MSLICE (fusable) in xehp_init_mcr()
1300 * - LNCF (sub-unit within mslice; always present if mslice is present) in xehp_init_mcr()
1305 * a suitable GSLICE, then we can just re-use the default value and in xehp_init_mcr()
1319 slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask, in xehp_init_mcr()
1326 for_each_set_bit(i, >->info.mslice_mask, GEN12_MAX_MSLICES) in xehp_init_mcr()
1335 gt->steering_table[LNCF] = NULL; in xehp_init_mcr()
1339 if (slice_mask & gt->info.mslice_mask) { in xehp_init_mcr()
1340 slice_mask &= gt->info.mslice_mask; in xehp_init_mcr()
1341 gt->steering_table[MSLICE] = NULL; in xehp_init_mcr()
1344 if (IS_XEHPSDV(gt->i915) && slice_mask & BIT(0)) in xehp_init_mcr()
1345 gt->steering_table[GAM] = NULL; in xehp_init_mcr()
1356 * DG2-G10, any value in the steering registers will work fine since in xehp_init_mcr()
1357 * all instances are present, but DG2-G11 only has SQIDI instances at in xehp_init_mcr()
1369 if (IS_DG2(gt->i915)) in xehp_init_mcr()
1380 * non-fused-off DSS. All other types of MCR registers will be in pvc_init_mcr()
1383 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0); in pvc_init_mcr()
1390 struct drm_i915_private *i915 = gt->i915; in icl_gt_workarounds_init()
1454 * Though there are per-engine instances of these registers,
1455 * they retain their value through engine resets and should
1457 * the engine-specific workaround list.
1462 struct intel_engine_cs *engine; in wa_14011060649() local
1465 for_each_engine(engine, gt, id) { in wa_14011060649()
1466 if (engine->class != VIDEO_DECODE_CLASS || in wa_14011060649()
1467 (engine->instance % 2)) in wa_14011060649()
1470 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), in wa_14011060649()
1480 /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */ in gen12_gt_workarounds_init()
1483 /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */ in gen12_gt_workarounds_init()
1509 /* Empirical testing shows this register is unaffected by engine reset. */ in dg1_gt_workarounds_init()
1516 struct drm_i915_private *i915 = gt->i915; in xehpsdv_gt_workarounds_init()
1591 if (IS_DG2_G10(gt->i915)) { in dg2_gt_workarounds_init()
1671 struct intel_engine_cs *engine; in wa_16021867713() local
1674 for_each_engine(engine, gt, id) in wa_16021867713()
1675 if (engine->class == VIDEO_DECODE_CLASS) in wa_16021867713()
1676 wa_write_or(wal, VDBOX_CGCTL3F1C(engine->mmio_base), in wa_16021867713()
1707 * engine resets and also are not part of any engine's register state context.
1708 * I.e., settings that only need to be re-applied in the event of a full GT
1718 if (IS_PONTEVECCHIO(gt->i915)) { in gt_tuning_settings()
1724 if (IS_DG2(gt->i915)) { in gt_tuning_settings()
1733 struct drm_i915_private *i915 = gt->i915; in gt_init_workarounds()
1737 if (gt->type == GT_MEDIA) { in gt_init_workarounds()
1792 struct i915_wa_list *wal = >->wa_list; in intel_gt_init_workarounds()
1803 if ((cur ^ wa->set) & wa->read) { in wa_verify()
1806 name, from, i915_mmio_reg_offset(wa->reg), in wa_verify()
1807 cur, cur & wa->read, wa->set & wa->read); in wa_verify()
1817 struct intel_gt *gt = wal->gt; in wa_list_apply()
1818 struct intel_uncore *uncore = gt->uncore; in wa_list_apply()
1824 if (!wal->count) in wa_list_apply()
1830 spin_lock(&uncore->lock); in wa_list_apply()
1833 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_apply()
1836 /* open-coded rmw due to steering */ in wa_list_apply()
1837 if (wa->clr) in wa_list_apply()
1838 old = wa->is_mcr ? in wa_list_apply()
1839 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_apply()
1840 intel_uncore_read_fw(uncore, wa->reg); in wa_list_apply()
1841 val = (old & ~wa->clr) | wa->set; in wa_list_apply()
1842 if (val != old || !wa->clr) { in wa_list_apply()
1843 if (wa->is_mcr) in wa_list_apply()
1844 intel_gt_mcr_multicast_write_fw(gt, wa->mcr_reg, val); in wa_list_apply()
1846 intel_uncore_write_fw(uncore, wa->reg, val); in wa_list_apply()
1850 u32 val = wa->is_mcr ? in wa_list_apply()
1851 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_apply()
1852 intel_uncore_read_fw(uncore, wa->reg); in wa_list_apply()
1854 wa_verify(gt, wa, val, wal->name, "application"); in wa_list_apply()
1859 spin_unlock(&uncore->lock); in wa_list_apply()
1865 wa_list_apply(>->wa_list); in intel_gt_apply_workarounds()
1872 struct intel_uncore *uncore = gt->uncore; in wa_list_verify()
1882 spin_lock(&uncore->lock); in wa_list_verify()
1885 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wa_list_verify()
1886 ok &= wa_verify(wal->gt, wa, wa->is_mcr ? in wa_list_verify()
1887 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : in wa_list_verify()
1888 intel_uncore_read_fw(uncore, wa->reg), in wa_list_verify()
1889 wal->name, from); in wa_list_verify()
1892 spin_unlock(&uncore->lock); in wa_list_verify()
1900 return wa_list_verify(gt, >->wa_list, from); in intel_gt_verify_workarounds()
1925 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_reg_ext()
1943 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_mcr_reg_ext()
1980 static void skl_whitelist_build(struct intel_engine_cs *engine) in skl_whitelist_build() argument
1982 struct i915_wa_list *w = &engine->whitelist; in skl_whitelist_build()
1984 if (engine->class != RENDER_CLASS) in skl_whitelist_build()
1993 static void bxt_whitelist_build(struct intel_engine_cs *engine) in bxt_whitelist_build() argument
1995 if (engine->class != RENDER_CLASS) in bxt_whitelist_build()
1998 gen9_whitelist_build(&engine->whitelist); in bxt_whitelist_build()
2001 static void kbl_whitelist_build(struct intel_engine_cs *engine) in kbl_whitelist_build() argument
2003 struct i915_wa_list *w = &engine->whitelist; in kbl_whitelist_build()
2005 if (engine->class != RENDER_CLASS) in kbl_whitelist_build()
2014 static void glk_whitelist_build(struct intel_engine_cs *engine) in glk_whitelist_build() argument
2016 struct i915_wa_list *w = &engine->whitelist; in glk_whitelist_build()
2018 if (engine->class != RENDER_CLASS) in glk_whitelist_build()
2027 static void cfl_whitelist_build(struct intel_engine_cs *engine) in cfl_whitelist_build() argument
2029 struct i915_wa_list *w = &engine->whitelist; in cfl_whitelist_build()
2031 if (engine->class != RENDER_CLASS) in cfl_whitelist_build()
2040 * - PS_INVOCATION_COUNT in cfl_whitelist_build()
2041 * - PS_INVOCATION_COUNT_UDW in cfl_whitelist_build()
2042 * - PS_DEPTH_COUNT in cfl_whitelist_build()
2043 * - PS_DEPTH_COUNT_UDW in cfl_whitelist_build()
2050 static void allow_read_ctx_timestamp(struct intel_engine_cs *engine) in allow_read_ctx_timestamp() argument
2052 struct i915_wa_list *w = &engine->whitelist; in allow_read_ctx_timestamp()
2054 if (engine->class != RENDER_CLASS) in allow_read_ctx_timestamp()
2056 RING_CTX_TIMESTAMP(engine->mmio_base), in allow_read_ctx_timestamp()
2060 static void cml_whitelist_build(struct intel_engine_cs *engine) in cml_whitelist_build() argument
2062 allow_read_ctx_timestamp(engine); in cml_whitelist_build()
2064 cfl_whitelist_build(engine); in cml_whitelist_build()
2067 static void icl_whitelist_build(struct intel_engine_cs *engine) in icl_whitelist_build() argument
2069 struct i915_wa_list *w = &engine->whitelist; in icl_whitelist_build()
2071 allow_read_ctx_timestamp(engine); in icl_whitelist_build()
2073 switch (engine->class) { in icl_whitelist_build()
2088 * - PS_INVOCATION_COUNT in icl_whitelist_build()
2089 * - PS_INVOCATION_COUNT_UDW in icl_whitelist_build()
2090 * - PS_DEPTH_COUNT in icl_whitelist_build()
2091 * - PS_DEPTH_COUNT_UDW in icl_whitelist_build()
2100 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base), in icl_whitelist_build()
2103 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base), in icl_whitelist_build()
2106 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base), in icl_whitelist_build()
2115 static void tgl_whitelist_build(struct intel_engine_cs *engine) in tgl_whitelist_build() argument
2117 struct i915_wa_list *w = &engine->whitelist; in tgl_whitelist_build()
2119 allow_read_ctx_timestamp(engine); in tgl_whitelist_build()
2121 switch (engine->class) { in tgl_whitelist_build()
2128 * - PS_INVOCATION_COUNT in tgl_whitelist_build()
2129 * - PS_INVOCATION_COUNT_UDW in tgl_whitelist_build()
2130 * - PS_DEPTH_COUNT in tgl_whitelist_build()
2131 * - PS_DEPTH_COUNT_UDW in tgl_whitelist_build()
2140 * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p in tgl_whitelist_build()
2156 static void dg2_whitelist_build(struct intel_engine_cs *engine) in dg2_whitelist_build() argument
2158 struct i915_wa_list *w = &engine->whitelist; in dg2_whitelist_build()
2160 switch (engine->class) { in dg2_whitelist_build()
2171 static void blacklist_trtt(struct intel_engine_cs *engine) in blacklist_trtt() argument
2173 struct i915_wa_list *w = &engine->whitelist; in blacklist_trtt()
2179 * we cover the entire range on each engine. in blacklist_trtt()
2189 static void pvc_whitelist_build(struct intel_engine_cs *engine) in pvc_whitelist_build() argument
2192 blacklist_trtt(engine); in pvc_whitelist_build()
2195 static void xelpg_whitelist_build(struct intel_engine_cs *engine) in xelpg_whitelist_build() argument
2197 struct i915_wa_list *w = &engine->whitelist; in xelpg_whitelist_build()
2199 switch (engine->class) { in xelpg_whitelist_build()
2210 void intel_engine_init_whitelist(struct intel_engine_cs *engine) in intel_engine_init_whitelist() argument
2212 struct drm_i915_private *i915 = engine->i915; in intel_engine_init_whitelist()
2213 struct i915_wa_list *w = &engine->whitelist; in intel_engine_init_whitelist()
2215 wa_init_start(w, engine->gt, "whitelist", engine->name); in intel_engine_init_whitelist()
2217 if (engine->gt->type == GT_MEDIA) in intel_engine_init_whitelist()
2219 else if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 71))) in intel_engine_init_whitelist()
2220 xelpg_whitelist_build(engine); in intel_engine_init_whitelist()
2222 pvc_whitelist_build(engine); in intel_engine_init_whitelist()
2224 dg2_whitelist_build(engine); in intel_engine_init_whitelist()
2228 tgl_whitelist_build(engine); in intel_engine_init_whitelist()
2230 icl_whitelist_build(engine); in intel_engine_init_whitelist()
2232 cml_whitelist_build(engine); in intel_engine_init_whitelist()
2234 cfl_whitelist_build(engine); in intel_engine_init_whitelist()
2236 glk_whitelist_build(engine); in intel_engine_init_whitelist()
2238 kbl_whitelist_build(engine); in intel_engine_init_whitelist()
2240 bxt_whitelist_build(engine); in intel_engine_init_whitelist()
2242 skl_whitelist_build(engine); in intel_engine_init_whitelist()
2251 void intel_engine_apply_whitelist(struct intel_engine_cs *engine) in intel_engine_apply_whitelist() argument
2253 const struct i915_wa_list *wal = &engine->whitelist; in intel_engine_apply_whitelist()
2254 struct intel_uncore *uncore = engine->uncore; in intel_engine_apply_whitelist()
2255 const u32 base = engine->mmio_base; in intel_engine_apply_whitelist()
2259 if (!wal->count) in intel_engine_apply_whitelist()
2262 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in intel_engine_apply_whitelist()
2265 i915_mmio_reg_offset(wa->reg)); in intel_engine_apply_whitelist()
2282 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_fake_wa_init() argument
2292 * spec in certain circumstances on specific platforms. in engine_fake_wa_init()
2294 if (GRAPHICS_VER(engine->i915) >= 12) { in engine_fake_wa_init()
2295 mocs_r = engine->gt->mocs.uc_index; in engine_fake_wa_init()
2296 mocs_w = engine->gt->mocs.uc_index; in engine_fake_wa_init()
2298 if (HAS_L3_CCS_READ(engine->i915) && in engine_fake_wa_init()
2299 engine->class == COMPUTE_CLASS) { in engine_fake_wa_init()
2300 mocs_r = engine->gt->mocs.wb_index; in engine_fake_wa_init()
2308 drm_WARN_ON(&engine->i915->drm, mocs_r == 0); in engine_fake_wa_init()
2312 RING_CMD_CCTL(engine->mmio_base), in engine_fake_wa_init()
2319 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in rcs_engine_wa_init() argument
2321 struct drm_i915_private *i915 = engine->i915; in rcs_engine_wa_init()
2322 struct intel_gt *gt = engine->gt; in rcs_engine_wa_init()
2366 * Wa_1606700617:tgl,dg1,adl-p in rcs_engine_wa_init()
2367 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p in rcs_engine_wa_init()
2368 * Wa_14010826681:tgl,dg1,rkl,adl-p in rcs_engine_wa_init()
2378 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */ in rcs_engine_wa_init()
2385 * Wa_14010919138:rkl,dg1,adl-s,adl-p in rcs_engine_wa_init()
2390 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */ in rcs_engine_wa_init()
2411 * BSpec; some indicate this is an A0-only WA, others indicate in rcs_engine_wa_init()
2485 * Intel platforms that support fine-grained preemption (i.e., gen9 and in rcs_engine_wa_init()
2486 * beyond) allow the kernel-mode driver to choose between two different in rcs_engine_wa_init()
2491 * kernel-only register CS_DEBUG_MODE1 (0x20EC). Any granularity in rcs_engine_wa_init()
2492 * and settings chosen by the kernel-mode driver will apply to all in rcs_engine_wa_init()
2496 * Preemption settings are controlled on a per-context basis via in rcs_engine_wa_init()
2504 * that name is somewhat misleading as other non-granularity in rcs_engine_wa_init()
2510 * userspace developed before object-level preemption was enabled would in rcs_engine_wa_init()
2513 * object-level preemption disabled by default (see in rcs_engine_wa_init()
2515 * userspace drivers could opt-in to object-level preemption as they in rcs_engine_wa_init()
2516 * saw fit. For post-gen9 platforms, we continue to utilize Option 2; in rcs_engine_wa_init()
2523 * - Wa_14015141709: On DG2 and early steppings of MTL, in rcs_engine_wa_init()
2524 * CS_CHICKEN1[0] does not disable object-level preemption as in rcs_engine_wa_init()
2527 * to disable object-level preemption on these platforms/steppings in rcs_engine_wa_init()
2530 * - Wa_16013994831: May require that userspace program in rcs_engine_wa_init()
2787 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in xcs_engine_wa_init() argument
2789 struct drm_i915_private *i915 = engine->i915; in xcs_engine_wa_init()
2794 RING_SEMA_WAIT_POLL(engine->mmio_base), in xcs_engine_wa_init()
2798 if (NEEDS_FASTCOLOR_BLT_WABB(engine)) in xcs_engine_wa_init()
2799 wa_masked_field_set(wal, ECOSKPD(engine->mmio_base), in xcs_engine_wa_init()
2805 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in ccs_engine_wa_init() argument
2807 if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) { in ccs_engine_wa_init()
2817 * the GuC save/restore lists, re-applied at the right times, and checked for
2821 * part of an engine's register state context. If a register is part of a
2829 struct drm_i915_private *i915 = gt->i915; in add_render_compute_tuning_settings()
2835 * This tuning setting proves beneficial only on ATS-M designs; the in add_render_compute_tuning_settings()
2839 if (INTEL_INFO(i915)->tuning_thread_rr_after_dep) in add_render_compute_tuning_settings()
2850 * specific engine. Since all render+compute engines get reset
2853 * here and then add them to just a single RCS or CCS engine's
2854 * workaround list (whichever engine has the XXXX flag).
2857 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in general_render_compute_wa_init() argument
2859 struct drm_i915_private *i915 = engine->i915; in general_render_compute_wa_init()
2860 struct intel_gt *gt = engine->gt; in general_render_compute_wa_init()
2956 * Note that register 0xE420 is write-only and cannot be read in general_render_compute_wa_init()
2962 0 /* write-only, so skip validation */, in general_render_compute_wa_init()
2984 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_init_workarounds() argument
2986 if (GRAPHICS_VER(engine->i915) < 4) in engine_init_workarounds()
2989 engine_fake_wa_init(engine, wal); in engine_init_workarounds()
2993 * to a single RCS/CCS engine's workaround list since in engine_init_workarounds()
2996 if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) in engine_init_workarounds()
2997 general_render_compute_wa_init(engine, wal); in engine_init_workarounds()
2999 if (engine->class == COMPUTE_CLASS) in engine_init_workarounds()
3000 ccs_engine_wa_init(engine, wal); in engine_init_workarounds()
3001 else if (engine->class == RENDER_CLASS) in engine_init_workarounds()
3002 rcs_engine_wa_init(engine, wal); in engine_init_workarounds()
3004 xcs_engine_wa_init(engine, wal); in engine_init_workarounds()
3007 void intel_engine_init_workarounds(struct intel_engine_cs *engine) in intel_engine_init_workarounds() argument
3009 struct i915_wa_list *wal = &engine->wa_list; in intel_engine_init_workarounds()
3011 wa_init_start(wal, engine->gt, "engine", engine->name); in intel_engine_init_workarounds()
3012 engine_init_workarounds(engine, wal); in intel_engine_init_workarounds()
3016 void intel_engine_apply_workarounds(struct intel_engine_cs *engine) in intel_engine_apply_workarounds() argument
3018 wa_list_apply(&engine->wa_list); in intel_engine_apply_workarounds()
3088 struct drm_i915_private *i915 = rq->i915; in wa_list_srm()
3097 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
3098 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg))) in wa_list_srm()
3106 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
3107 u32 offset = i915_mmio_reg_offset(wa->reg); in wa_list_srm()
3134 if (!wal->count) in engine_wa_list_verify()
3137 vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm, in engine_wa_list_verify()
3138 wal->count * sizeof(u32)); in engine_wa_list_verify()
3142 intel_engine_pm_get(ce->engine); in engine_wa_list_verify()
3145 err = i915_gem_object_lock(vma->obj, &ww); in engine_wa_list_verify()
3175 err = -ETIME; in engine_wa_list_verify()
3179 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); in engine_wa_list_verify()
3186 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in engine_wa_list_verify()
3187 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg))) in engine_wa_list_verify()
3190 if (!wa_verify(wal->gt, wa, results[i], wal->name, from)) in engine_wa_list_verify()
3191 err = -ENXIO; in engine_wa_list_verify()
3194 i915_gem_object_unpin_map(vma->obj); in engine_wa_list_verify()
3203 if (err == -EDEADLK) { in engine_wa_list_verify()
3209 intel_engine_pm_put(ce->engine); in engine_wa_list_verify()
3214 int intel_engine_verify_workarounds(struct intel_engine_cs *engine, in intel_engine_verify_workarounds() argument
3217 return engine_wa_list_verify(engine->kernel_context, in intel_engine_verify_workarounds()
3218 &engine->wa_list, in intel_engine_verify_workarounds()