47 static void set_hwstam(struct intel_engine_cs *engine, u32 mask)  in set_hwstam()  argument
53 	if (engine->class == RENDER_CLASS) {  in set_hwstam()
54 		if (INTEL_GEN(engine->i915) >= 6)  in set_hwstam()
60 	intel_engine_set_hwsp_writemask(engine, mask);  in set_hwstam()
63 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)  in set_hws_pga()  argument
68 	if (INTEL_GEN(engine->i915) >= 4)  in set_hws_pga()
71 	intel_uncore_write(engine->uncore, HWS_PGA, addr);  in set_hws_pga()
74 static struct page *status_page(struct intel_engine_cs *engine)  in status_page()  argument
76 	struct drm_i915_gem_object *obj = engine->status_page.vma->obj;  in status_page()
82 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)  in ring_setup_phys_status_page()  argument
84 	set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));  in ring_setup_phys_status_page()
85 	set_hwstam(engine, ~0u);  in ring_setup_phys_status_page()
88 static void set_hwsp(struct intel_engine_cs *engine, u32 offset)  in set_hwsp()  argument
96 	if (IS_GEN(engine->i915, 7)) {  in set_hwsp()
97 		switch (engine->id) {  in set_hwsp()
103 			GEM_BUG_ON(engine->id);  in set_hwsp()
118 	} else if (IS_GEN(engine->i915, 6)) {  in set_hwsp()
119 		hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);  in set_hwsp()
121 		hwsp = RING_HWS_PGA(engine->mmio_base);  in set_hwsp()
124 	intel_uncore_write(engine->uncore, hwsp, offset);  in set_hwsp()
125 	intel_uncore_posting_read(engine->uncore, hwsp);  in set_hwsp()
128 static void flush_cs_tlb(struct intel_engine_cs *engine)  in flush_cs_tlb()  argument
130 	struct drm_i915_private *dev_priv = engine->i915;  in flush_cs_tlb()
137 		    (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);  in flush_cs_tlb()
139 	ENGINE_WRITE(engine, RING_INSTPM,  in flush_cs_tlb()
142 	if (intel_wait_for_register(engine->uncore,  in flush_cs_tlb()
143 				    RING_INSTPM(engine->mmio_base),  in flush_cs_tlb()
148 			engine->name);  in flush_cs_tlb()
151 static void ring_setup_status_page(struct intel_engine_cs *engine)  in ring_setup_status_page()  argument
153 	set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));  in ring_setup_status_page()
154 	set_hwstam(engine, ~0u);  in ring_setup_status_page()
156 	flush_cs_tlb(engine);  in ring_setup_status_page()
159 static bool stop_ring(struct intel_engine_cs *engine)  in stop_ring()  argument
161 	struct drm_i915_private *dev_priv = engine->i915;  in stop_ring()
164 		ENGINE_WRITE(engine,  in stop_ring()
166 		if (intel_wait_for_register(engine->uncore,  in stop_ring()
167 					    RING_MI_MODE(engine->mmio_base),  in stop_ring()
173 				engine->name);  in stop_ring()
180 			if (ENGINE_READ(engine, RING_HEAD) !=  in stop_ring()
181 			    ENGINE_READ(engine, RING_TAIL))  in stop_ring()
186 	ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL));  in stop_ring()
188 	ENGINE_WRITE(engine, RING_HEAD, 0);  in stop_ring()
189 	ENGINE_WRITE(engine, RING_TAIL, 0);  in stop_ring()
192 	ENGINE_WRITE(engine, RING_CTL, 0);  in stop_ring()
194 	return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0;  in stop_ring()
210 static void set_pp_dir(struct intel_engine_cs *engine)  in set_pp_dir()  argument
212 	struct i915_address_space *vm = vm_alias(engine->gt->vm);  in set_pp_dir()
215 		ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G);  in set_pp_dir()
216 		ENGINE_WRITE(engine, RING_PP_DIR_BASE, pp_dir(vm));  in set_pp_dir()
220 static int xcs_resume(struct intel_engine_cs *engine)  in xcs_resume()  argument
222 	struct drm_i915_private *dev_priv = engine->i915;  in xcs_resume()
223 	struct intel_ring *ring = engine->legacy.ring;  in xcs_resume()
226 	ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",  in xcs_resume()
229 	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);  in xcs_resume()
232 	if (!stop_ring(engine)) {  in xcs_resume()
236 			engine->name,  in xcs_resume()
237 			ENGINE_READ(engine, RING_CTL),  in xcs_resume()
238 			ENGINE_READ(engine, RING_HEAD),  in xcs_resume()
239 			ENGINE_READ(engine, RING_TAIL),  in xcs_resume()
240 			ENGINE_READ(engine, RING_START));  in xcs_resume()
242 		if (!stop_ring(engine)) {  in xcs_resume()
246 				engine->name,  in xcs_resume()
247 				ENGINE_READ(engine, RING_CTL),  in xcs_resume()
248 				ENGINE_READ(engine, RING_HEAD),  in xcs_resume()
249 				ENGINE_READ(engine, RING_TAIL),  in xcs_resume()
250 				ENGINE_READ(engine, RING_START));  in xcs_resume()
257 		ring_setup_phys_status_page(engine);  in xcs_resume()
259 		ring_setup_status_page(engine);  in xcs_resume()
261 	intel_breadcrumbs_reset(engine->breadcrumbs);  in xcs_resume()
264 	ENGINE_POSTING_READ(engine, RING_HEAD);  in xcs_resume()
272 	ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma));  in xcs_resume()
279 	set_pp_dir(engine);  in xcs_resume()
282 	ENGINE_WRITE(engine, RING_HEAD, ring->head);  in xcs_resume()
283 	ENGINE_WRITE(engine, RING_TAIL, ring->head);  in xcs_resume()
284 	ENGINE_POSTING_READ(engine, RING_TAIL);  in xcs_resume()
286 	ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID);  in xcs_resume()
289 	if (intel_wait_for_register(engine->uncore,  in xcs_resume()
290 				    RING_CTL(engine->mmio_base),  in xcs_resume()
295 			  engine->name,  in xcs_resume()
296 			  ENGINE_READ(engine, RING_CTL),  in xcs_resume()
297 			  ENGINE_READ(engine, RING_CTL) & RING_VALID,  in xcs_resume()
298 			  ENGINE_READ(engine, RING_HEAD), ring->head,  in xcs_resume()
299 			  ENGINE_READ(engine, RING_TAIL), ring->tail,  in xcs_resume()
300 			  ENGINE_READ(engine, RING_START),  in xcs_resume()
307 		ENGINE_WRITE(engine,  in xcs_resume()
312 		ENGINE_WRITE(engine, RING_TAIL, ring->tail);  in xcs_resume()
313 		ENGINE_POSTING_READ(engine, RING_TAIL);  in xcs_resume()
317 	intel_engine_signal_breadcrumbs(engine);  in xcs_resume()
319 	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);  in xcs_resume()
324 static void reset_prepare(struct intel_engine_cs *engine)  in reset_prepare()  argument
326 	struct intel_uncore *uncore = engine->uncore;  in reset_prepare()
327 	const u32 base = engine->mmio_base;  in reset_prepare()
343 	ENGINE_TRACE(engine, "\n");  in reset_prepare()
345 	if (intel_engine_stop_cs(engine))  in reset_prepare()
346 		ENGINE_TRACE(engine, "timed out on STOP_RING\n");  in reset_prepare()
362 		ENGINE_TRACE(engine, "ring head [%x] not parked\n",  in reset_prepare()
366 static void reset_rewind(struct intel_engine_cs *engine, bool stalled)  in reset_rewind()  argument
373 	spin_lock_irqsave(&engine->active.lock, flags);  in reset_rewind()
374 	list_for_each_entry(pos, &engine->active.requests, sched.link) {  in reset_rewind()
382 	 * The guilty request will get skipped on a hung engine.  in reset_rewind()
421 		GEM_BUG_ON(rq->ring != engine->legacy.ring);  in reset_rewind()
424 		head = engine->legacy.ring->tail;  in reset_rewind()
426 	engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);  in reset_rewind()
428 	spin_unlock_irqrestore(&engine->active.lock, flags);  in reset_rewind()
431 static void reset_finish(struct intel_engine_cs *engine)  in reset_finish()  argument
435 static void reset_cancel(struct intel_engine_cs *engine)  in reset_cancel()  argument
440 	spin_lock_irqsave(&engine->active.lock, flags);  in reset_cancel()
443 	list_for_each_entry(request, &engine->active.requests, sched.link) {  in reset_cancel()
450 	spin_unlock_irqrestore(&engine->active.lock, flags);  in reset_cancel()
458 	ENGINE_WRITE(request->engine, RING_TAIL,  in i9xx_submit_request()
513 alloc_context_vma(struct intel_engine_cs *engine)  in alloc_context_vma()  argument
515 	struct drm_i915_private *i915 = engine->i915;  in alloc_context_vma()
520 	obj = i915_gem_object_create_shmem(i915, engine->context_size);  in alloc_context_vma()
542 	if (engine->default_state) {  in alloc_context_vma()
551 		shmem_read(engine->default_state, 0,  in alloc_context_vma()
552 			   vaddr, engine->context_size);  in alloc_context_vma()
558 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in alloc_context_vma()
573 	struct intel_engine_cs *engine = ce->engine;  in ring_context_alloc()  local
576 	GEM_BUG_ON(!engine->legacy.ring);  in ring_context_alloc()
577 	ce->ring = engine->legacy.ring;  in ring_context_alloc()
578 	ce->timeline = intel_timeline_get(engine->legacy.timeline);  in ring_context_alloc()
581 	if (engine->context_size) {  in ring_context_alloc()
584 		vma = alloc_context_vma(engine);  in ring_context_alloc()
589 		if (engine->default_state)  in ring_context_alloc()
625 	const struct intel_engine_cs * const engine = rq->engine;  in load_pd_dir()  local
633 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));  in load_pd_dir()
637 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));  in load_pd_dir()
642 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));  in load_pd_dir()
643 	*cs++ = intel_gt_scratch_offset(engine->gt,  in load_pd_dir()
647 	*cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));  in load_pd_dir()
652 	return rq->engine->emit_flush(rq, EMIT_FLUSH);  in load_pd_dir()
659 	struct intel_engine_cs *engine = rq->engine;  in mi_set_context()  local
660 	struct drm_i915_private *i915 = engine->i915;  in mi_set_context()
663 		IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0;  in mi_set_context()
691 			for_each_engine(signaller, engine->gt, id) {  in mi_set_context()
692 				if (signaller == engine)  in mi_set_context()
725 		*cs++ = i915_ggtt_offset(engine->kernel_context->state) |  in mi_set_context()
745 			for_each_engine(signaller, engine->gt, id) {  in mi_set_context()
746 				if (signaller == engine)  in mi_set_context()
758 			*cs++ = intel_gt_scratch_offset(engine->gt,  in mi_set_context()
774 	u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice];  in remap_l3_slice()
828 	ret = rq->engine->emit_flush(rq, EMIT_FLUSH);  in switch_mm()
844 	return rq->engine->emit_flush(rq, EMIT_INVALIDATE);  in switch_mm()
849 	struct intel_engine_cs *engine = rq->engine;  in clear_residuals()  local
852 	ret = switch_mm(rq, vm_alias(engine->kernel_context->vm));  in clear_residuals()
856 	if (engine->kernel_context->state) {  in clear_residuals()
858 				     engine->kernel_context,  in clear_residuals()
864 	ret = engine->emit_bb_start(rq,  in clear_residuals()
865 				    engine->wa_ctx.vma->node.start, 0,  in clear_residuals()
870 	ret = engine->emit_flush(rq, EMIT_FLUSH);  in clear_residuals()
875 	return engine->emit_flush(rq, EMIT_INVALIDATE);  in clear_residuals()
880 	struct intel_engine_cs *engine = rq->engine;  in switch_context()  local
885 	GEM_BUG_ON(HAS_EXECLISTS(engine->i915));  in switch_context()
887 	if (engine->wa_ctx.vma && ce != engine->kernel_context) {  in switch_context()
888 		if (engine->wa_ctx.vma->private != ce) {  in switch_context()
893 			residuals = &engine->wa_ctx.vma->private;  in switch_context()
904 		GEM_BUG_ON(engine->id != RCS0);  in switch_context()
957 	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);  in ring_request_alloc()
971 	struct intel_uncore *uncore = request->engine->uncore;  in gen6_bsd_submit_request()
1007 static void i9xx_set_default_submission(struct intel_engine_cs *engine)  in i9xx_set_default_submission()  argument
1009 	engine->submit_request = i9xx_submit_request;  in i9xx_set_default_submission()
1011 	engine->park = NULL;  in i9xx_set_default_submission()
1012 	engine->unpark = NULL;  in i9xx_set_default_submission()
1015 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)  in gen6_bsd_set_default_submission()  argument
1017 	i9xx_set_default_submission(engine);  in gen6_bsd_set_default_submission()
1018 	engine->submit_request = gen6_bsd_submit_request;  in gen6_bsd_set_default_submission()
1021 static void ring_release(struct intel_engine_cs *engine)  in ring_release()  argument
1023 	struct drm_i915_private *dev_priv = engine->i915;  in ring_release()
1026 		    (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);  in ring_release()
1028 	intel_engine_cleanup_common(engine);  in ring_release()
1030 	if (engine->wa_ctx.vma) {  in ring_release()
1031 		intel_context_put(engine->wa_ctx.vma->private);  in ring_release()
1032 		i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);  in ring_release()
1035 	intel_ring_unpin(engine->legacy.ring);  in ring_release()
1036 	intel_ring_put(engine->legacy.ring);  in ring_release()
1038 	intel_timeline_unpin(engine->legacy.timeline);  in ring_release()
1039 	intel_timeline_put(engine->legacy.timeline);  in ring_release()
1042 static void setup_irq(struct intel_engine_cs *engine)  in setup_irq()  argument
1044 	struct drm_i915_private *i915 = engine->i915;  in setup_irq()
1047 		engine->irq_enable = gen6_irq_enable;  in setup_irq()
1048 		engine->irq_disable = gen6_irq_disable;  in setup_irq()
1050 		engine->irq_enable = gen5_irq_enable;  in setup_irq()
1051 		engine->irq_disable = gen5_irq_disable;  in setup_irq()
1053 		engine->irq_enable = gen3_irq_enable;  in setup_irq()
1054 		engine->irq_disable = gen3_irq_disable;  in setup_irq()
1056 		engine->irq_enable = gen2_irq_enable;  in setup_irq()
1057 		engine->irq_disable = gen2_irq_disable;  in setup_irq()
1061 static void setup_common(struct intel_engine_cs *engine)  in setup_common()  argument
1063 	struct drm_i915_private *i915 = engine->i915;  in setup_common()
1068 	setup_irq(engine);  in setup_common()
1070 	engine->resume = xcs_resume;  in setup_common()
1071 	engine->reset.prepare = reset_prepare;  in setup_common()
1072 	engine->reset.rewind = reset_rewind;  in setup_common()
1073 	engine->reset.cancel = reset_cancel;  in setup_common()
1074 	engine->reset.finish = reset_finish;  in setup_common()
1076 	engine->cops = &ring_context_ops;  in setup_common()
1077 	engine->request_alloc = ring_request_alloc;  in setup_common()
1082 	 * engine->emit_init_breadcrumb().  in setup_common()
1084 	engine->emit_fini_breadcrumb = gen3_emit_breadcrumb;  in setup_common()
1086 		engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;  in setup_common()
1088 	engine->set_default_submission = i9xx_set_default_submission;  in setup_common()
1091 		engine->emit_bb_start = gen6_emit_bb_start;  in setup_common()
1093 		engine->emit_bb_start = gen4_emit_bb_start;  in setup_common()
1095 		engine->emit_bb_start = i830_emit_bb_start;  in setup_common()
1097 		engine->emit_bb_start = gen3_emit_bb_start;  in setup_common()
1100 static void setup_rcs(struct intel_engine_cs *engine)  in setup_rcs()  argument
1102 	struct drm_i915_private *i915 = engine->i915;  in setup_rcs()
1105 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;  in setup_rcs()
1107 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;  in setup_rcs()
1110 		engine->emit_flush = gen7_emit_flush_rcs;  in setup_rcs()
1111 		engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs;  in setup_rcs()
1113 		engine->emit_flush = gen6_emit_flush_rcs;  in setup_rcs()
1114 		engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs;  in setup_rcs()
1116 		engine->emit_flush = gen4_emit_flush_rcs;  in setup_rcs()
1119 			engine->emit_flush = gen2_emit_flush;  in setup_rcs()
1121 			engine->emit_flush = gen4_emit_flush_rcs;  in setup_rcs()
1122 		engine->irq_enable_mask = I915_USER_INTERRUPT;  in setup_rcs()
1126 		engine->emit_bb_start = hsw_emit_bb_start;  in setup_rcs()
1129 static void setup_vcs(struct intel_engine_cs *engine)  in setup_vcs()  argument
1131 	struct drm_i915_private *i915 = engine->i915;  in setup_vcs()
1136 			engine->set_default_submission = gen6_bsd_set_default_submission;  in setup_vcs()
1137 		engine->emit_flush = gen6_emit_flush_vcs;  in setup_vcs()
1138 		engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;  in setup_vcs()
1141 			engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;  in setup_vcs()
1143 			engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;  in setup_vcs()
1145 		engine->emit_flush = gen4_emit_flush_vcs;  in setup_vcs()
1147 			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;  in setup_vcs()
1149 			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;  in setup_vcs()
1153 static void setup_bcs(struct intel_engine_cs *engine)  in setup_bcs()  argument
1155 	struct drm_i915_private *i915 = engine->i915;  in setup_bcs()
1157 	engine->emit_flush = gen6_emit_flush_xcs;  in setup_bcs()
1158 	engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;  in setup_bcs()
1161 		engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;  in setup_bcs()
1163 		engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;  in setup_bcs()
1166 static void setup_vecs(struct intel_engine_cs *engine)  in setup_vecs()  argument
1168 	struct drm_i915_private *i915 = engine->i915;  in setup_vecs()
1172 	engine->emit_flush = gen6_emit_flush_xcs;  in setup_vecs()
1173 	engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;  in setup_vecs()
1174 	engine->irq_enable = hsw_irq_enable_vecs;  in setup_vecs()
1175 	engine->irq_disable = hsw_irq_disable_vecs;  in setup_vecs()
1177 	engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;  in setup_vecs()
1180 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,  in gen7_ctx_switch_bb_setup()  argument
1183 	return gen7_setup_clear_gpr_bb(engine, vma);  in gen7_ctx_switch_bb_setup()
1186 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)  in gen7_ctx_switch_bb_init()  argument
1193 	size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);  in gen7_ctx_switch_bb_init()
1198 	obj = i915_gem_object_create_internal(engine->i915, size);  in gen7_ctx_switch_bb_init()
1202 	vma = i915_vma_instance(obj, engine->gt->vm, NULL);  in gen7_ctx_switch_bb_init()
1208 	vma->private = intel_context_create(engine); /* dummy residuals */  in gen7_ctx_switch_bb_init()
1222 	err = gen7_ctx_switch_bb_setup(engine, vma);  in gen7_ctx_switch_bb_init()
1226 	engine->wa_ctx.vma = vma;  in gen7_ctx_switch_bb_init()
1238 int intel_ring_submission_setup(struct intel_engine_cs *engine)  in intel_ring_submission_setup()  argument
1244 	setup_common(engine);  in intel_ring_submission_setup()
1246 	switch (engine->class) {  in intel_ring_submission_setup()
1248 		setup_rcs(engine);  in intel_ring_submission_setup()
1251 		setup_vcs(engine);  in intel_ring_submission_setup()
1254 		setup_bcs(engine);  in intel_ring_submission_setup()
1257 		setup_vecs(engine);  in intel_ring_submission_setup()
1260 		MISSING_CASE(engine->class);  in intel_ring_submission_setup()
1264 	timeline = intel_timeline_create_from_engine(engine,  in intel_ring_submission_setup()
1276 	ring = intel_engine_create_ring(engine, SZ_16K);  in intel_ring_submission_setup()
1286 	GEM_BUG_ON(engine->legacy.ring);  in intel_ring_submission_setup()
1287 	engine->legacy.ring = ring;  in intel_ring_submission_setup()
1288 	engine->legacy.timeline = timeline;  in intel_ring_submission_setup()
1290 	GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);  in intel_ring_submission_setup()
1292 	if (IS_HASWELL(engine->i915) && engine->class == RENDER_CLASS) {  in intel_ring_submission_setup()
1293 		err = gen7_ctx_switch_bb_init(engine);  in intel_ring_submission_setup()
1299 	engine->release = ring_release;  in intel_ring_submission_setup()
1312 	intel_engine_cleanup_common(engine);  in intel_ring_submission_setup()