44  * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
45  * contained there mean you don't need to do a ppgtt->switch_mm yourself,
51  * rings, the engine cs shifts to a new "ring buffer" with every context
62  * - One global default context.
63  * - One local default context for each opened fd.
64  * - One local extra context for each context create ioctl call.
66  * Now that ringbuffers belong per-context (and not per-engine, like before)
70  * - One ringbuffer per-engine inside each context.
71  * - One backing object per-engine inside each context.
91  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
107  * globally unique 20-bits submission ID.
193 	 * scheduling -- each real engine takes the next available request
201 	 * for the virtual engine, indexed by physical_engine->id.
209 	 * Keep track of bonded pairs -- restrictions upon on our selection
211 	 * If we receive a submit-fence from a master engine, we will only
246 	if (INTEL_GEN(engine->i915) >= 12)  in lrc_ring_mi_mode()
248 	else if (INTEL_GEN(engine->i915) >= 9)  in lrc_ring_mi_mode()
250 	else if (engine->class == RENDER_CLASS)  in lrc_ring_mi_mode()
253 		return -1;  in lrc_ring_mi_mode()
258 	if (INTEL_GEN(engine->i915) >= 12)  in lrc_ring_gpr0()
260 	else if (INTEL_GEN(engine->i915) >= 9)  in lrc_ring_gpr0()
262 	else if (engine->class == RENDER_CLASS)  in lrc_ring_gpr0()
265 		return -1;  in lrc_ring_gpr0()
270 	if (INTEL_GEN(engine->i915) >= 12)  in lrc_ring_wa_bb_per_ctx()
272 	else if (INTEL_GEN(engine->i915) >= 9 || engine->class == RENDER_CLASS)  in lrc_ring_wa_bb_per_ctx()
275 		return -1;  in lrc_ring_wa_bb_per_ctx()
280 	int x;  in lrc_ring_indirect_ptr()  local
282 	x = lrc_ring_wa_bb_per_ctx(engine);  in lrc_ring_indirect_ptr()
283 	if (x < 0)  in lrc_ring_indirect_ptr()
284 		return x;  in lrc_ring_indirect_ptr()
286 	return x + 2;  in lrc_ring_indirect_ptr()
291 	int x;  in lrc_ring_indirect_offset()  local
293 	x = lrc_ring_indirect_ptr(engine);  in lrc_ring_indirect_offset()
294 	if (x < 0)  in lrc_ring_indirect_offset()
295 		return x;  in lrc_ring_indirect_offset()
297 	return x + 2;  in lrc_ring_indirect_offset()
302 	if (engine->class != RENDER_CLASS)  in lrc_ring_cmd_buf_cctl()
303 		return -1;  in lrc_ring_cmd_buf_cctl()
305 	if (INTEL_GEN(engine->i915) >= 12)  in lrc_ring_cmd_buf_cctl()
307 	else if (INTEL_GEN(engine->i915) >= 11)  in lrc_ring_cmd_buf_cctl()
310 		return -1;  in lrc_ring_cmd_buf_cctl()
316 	switch (INTEL_GEN(engine->i915)) {  in lrc_ring_indirect_offset_default()
318 		MISSING_CASE(INTEL_GEN(engine->i915));  in lrc_ring_indirect_offset_default()
341 	GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1);  in lrc_ring_setup_indirect_ctx()
345 	GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1);  in lrc_ring_setup_indirect_ctx()
358 	return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]);  in intel_context_get_runtime()
368 	i915_request_set_error_once(rq, -EIO);  in mark_eio()
378 	list_for_each_entry_continue_reverse(rq, &tl->requests, link) {  in active_request()
391 	return (i915_ggtt_offset(engine->status_page.vma) +  in intel_hws_preempt_address()
400 	 * engine->emit_fini_breadcrumb. If the dword is true,  in ring_set_paused()
404 	engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state;  in ring_set_paused()
416 	return READ_ONCE(rq->sched.attr.priority);  in rq_prio()
442 	rb = rb_first_cached(&execlists->queue);  in queue_prio()
452 		return p->priority;  in queue_prio()
454 	return ((p->priority + 1) << I915_USER_PRIORITY_SHIFT) - ffs(p->used);  in queue_prio()
472 	 * a preempt-to-idle cycle.  in need_preempt()
484 	last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);  in need_preempt()
485 	if (engine->execlists.queue_priority_hint <= last_prio)  in need_preempt()
492 	if (!list_is_last(&rq->sched.link, &engine->active.requests) &&  in need_preempt()
498 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in need_preempt()
501 		if (engine == ve->siblings[0]) { /* only preempt one sibling */  in need_preempt()
505 			next = READ_ONCE(ve->request);  in need_preempt()
525 	return queue_prio(&engine->execlists) > last_prio;  in need_preempt()
553  *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx->desc_template)
554  *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
555  *      bits 32-52:    ctx ID, a globally unique tag (highest bit used by GuC)
556  *      bits 53-54:    mbz, reserved for use by hardware
557  *      bits 55-63:    group ID, currently unused and set to 0
561  *      bits 32-36:    reserved
562  *      bits 37-47:    SW context ID
565  *      bits 55-60:    SW counter
566  *      bits 61-63:    engine class
577 	if (i915_vm_is_4lvl(ce->vm))  in lrc_descriptor()
582 	if (IS_GEN(engine->i915, 8))  in lrc_descriptor()
585 	return i915_ggtt_offset(ce->state) | desc;  in lrc_descriptor()
597 #define NOP(x) (BIT(7) | (x))  in set_offsets()  argument
600 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200))  in set_offsets()  argument
601 #define REG16(x) \  in set_offsets()  argument
602 	(((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \  in set_offsets()
603 	(((x) >> 2) & 0x7f)  in set_offsets()
606 	const u32 base = engine->mmio_base;  in set_offsets()
626 		if (INTEL_GEN(engine->i915) >= 11)  in set_offsets()
645 		} while (--count);  in set_offsets()
653 		memset32(regs, MI_NOOP, count - dword_in_page(regs));  in set_offsets()
657 		if (INTEL_GEN(engine->i915) >= 10)  in set_offsets()
1085 	GEM_BUG_ON(INTEL_GEN(engine->i915) >= 12 &&  in reg_offsets()
1088 	if (engine->class == RENDER_CLASS) {  in reg_offsets()
1089 		if (INTEL_GEN(engine->i915) >= 12)  in reg_offsets()
1091 		else if (INTEL_GEN(engine->i915) >= 11)  in reg_offsets()
1093 		else if (INTEL_GEN(engine->i915) >= 9)  in reg_offsets()
1098 		if (INTEL_GEN(engine->i915) >= 12)  in reg_offsets()
1100 		else if (INTEL_GEN(engine->i915) >= 9)  in reg_offsets()
1114 	lockdep_assert_held(&engine->active.lock);  in __unwind_incomplete_requests()
1117 					 &engine->active.requests,  in __unwind_incomplete_requests()
1131 		if (likely(rq->execution_mask == engine->mask)) {  in __unwind_incomplete_requests()
1137 			GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));  in __unwind_incomplete_requests()
1139 			list_move(&rq->sched.link, pl);  in __unwind_incomplete_requests()
1140 			set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);  in __unwind_incomplete_requests()
1143 			if (intel_ring_direction(rq->ring,  in __unwind_incomplete_requests()
1144 						 rq->tail,  in __unwind_incomplete_requests()
1145 						 rq->ring->tail + 8) > 0)  in __unwind_incomplete_requests()
1146 				rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;  in __unwind_incomplete_requests()
1150 			struct intel_engine_cs *owner = rq->context->engine;  in __unwind_incomplete_requests()
1152 			WRITE_ONCE(rq->engine, owner);  in __unwind_incomplete_requests()
1153 			owner->submit_request(rq);  in __unwind_incomplete_requests()
1174 	 * Only used when GVT-g is enabled now. When GVT-g is disabled,  in execlists_context_status_change()
1175 	 * The compiler should eliminate this function as dead-code.  in execlists_context_status_change()
1180 	atomic_notifier_call_chain(&rq->engine->context_status_notifier,  in execlists_context_status_change()
1188 	if (atomic_add_unless(&engine->stats.active, 1, 0))  in intel_engine_context_in()
1191 	write_seqlock_irqsave(&engine->stats.lock, flags);  in intel_engine_context_in()
1192 	if (!atomic_add_unless(&engine->stats.active, 1, 0)) {  in intel_engine_context_in()
1193 		engine->stats.start = ktime_get();  in intel_engine_context_in()
1194 		atomic_inc(&engine->stats.active);  in intel_engine_context_in()
1196 	write_sequnlock_irqrestore(&engine->stats.lock, flags);  in intel_engine_context_in()
1203 	GEM_BUG_ON(!atomic_read(&engine->stats.active));  in intel_engine_context_out()
1205 	if (atomic_add_unless(&engine->stats.active, -1, 1))  in intel_engine_context_out()
1208 	write_seqlock_irqsave(&engine->stats.lock, flags);  in intel_engine_context_out()
1209 	if (atomic_dec_and_test(&engine->stats.active)) {  in intel_engine_context_out()
1210 		engine->stats.total =  in intel_engine_context_out()
1211 			ktime_add(engine->stats.total,  in intel_engine_context_out()
1212 				  ktime_sub(ktime_get(), engine->stats.start));  in intel_engine_context_out()
1214 	write_sequnlock_irqrestore(&engine->stats.lock, flags);  in intel_engine_context_out()
1221 	const struct intel_ring *ring = ce->ring;  in execlists_check_context()
1222 	u32 *regs = ce->lrc_reg_state;  in execlists_check_context()
1224 	int x;  in execlists_check_context()  local
1226 	if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) {  in execlists_check_context()
1227 		pr_err("%s: context submitted with incorrect RING_START [%08x], expected %08x\n",  in execlists_check_context()
1228 		       engine->name,  in execlists_check_context()
1230 		       i915_ggtt_offset(ring->vma));  in execlists_check_context()
1231 		regs[CTX_RING_START] = i915_ggtt_offset(ring->vma);  in execlists_check_context()
1236 	    (RING_CTL_SIZE(ring->size) | RING_VALID)) {  in execlists_check_context()
1237 		pr_err("%s: context submitted with incorrect RING_CTL [%08x], expected %08x\n",  in execlists_check_context()
1238 		       engine->name,  in execlists_check_context()
1240 		       (u32)(RING_CTL_SIZE(ring->size) | RING_VALID));  in execlists_check_context()
1241 		regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID;  in execlists_check_context()
1245 	x = lrc_ring_mi_mode(engine);  in execlists_check_context()
1246 	if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) {  in execlists_check_context()
1247 		pr_err("%s: context submitted with STOP_RING [%08x] in RING_MI_MODE\n",  in execlists_check_context()
1248 		       engine->name, regs[x + 1]);  in execlists_check_context()
1249 		regs[x + 1] &= ~STOP_RING;  in execlists_check_context()
1250 		regs[x + 1] |= STOP_RING << 16;  in execlists_check_context()
1262 	regs = memset(ce->lrc_reg_state, 0, engine->context_size - PAGE_SIZE);  in restore_default_state()
1263 	execlists_init_reg_state(regs, ce, engine, ce->ring, true);  in restore_default_state()
1265 	ce->runtime.last = intel_context_get_runtime(ce);  in restore_default_state()
1271 	struct intel_context * const ce = rq->context;  in reset_active()
1279 	 * In __i915_request_submit(), we apply the -EIO and remove the  in reset_active()
1285 	 * requests so that inter-timeline dependencies (i.e other timelines)  in reset_active()
1290 		     rq->fence.context, rq->fence.seqno);  in reset_active()
1294 		head = rq->tail;  in reset_active()
1296 		head = active_request(ce->timeline, rq)->head;  in reset_active()
1297 	head = intel_ring_wrap(ce->ring, head);  in reset_active()
1303 	/* We've switched away, so this should be a no-op, but intent matters */  in reset_active()
1304 	ce->lrc.desc |= CTX_DESC_FORCE_RESTORE;  in reset_active()
1310 	ce->runtime.num_underflow += dt < 0;  in st_update_runtime_underflow()
1311 	ce->runtime.max_underflow = max_t(u32, ce->runtime.max_underflow, -dt);  in st_update_runtime_underflow()
1323 	old = ce->runtime.last;  in intel_context_update_runtime()
1324 	ce->runtime.last = intel_context_get_runtime(ce);  in intel_context_update_runtime()
1325 	dt = ce->runtime.last - old;  in intel_context_update_runtime()
1329 			 old, ce->runtime.last, dt);  in intel_context_update_runtime()
1334 	ewma_runtime_add(&ce->runtime.avg, dt);  in intel_context_update_runtime()
1335 	ce->runtime.total += dt;  in intel_context_update_runtime()
1341 	struct intel_engine_cs * const engine = rq->engine;  in __execlists_schedule_in()
1342 	struct intel_context * const ce = rq->context;  in __execlists_schedule_in()
1352 	if (ce->tag) {  in __execlists_schedule_in()
1354 		GEM_BUG_ON(ce->tag <= BITS_PER_LONG);  in __execlists_schedule_in()
1355 		ce->lrc.ccid = ce->tag;  in __execlists_schedule_in()
1358 		unsigned int tag = ffs(READ_ONCE(engine->context_tag));  in __execlists_schedule_in()
1361 		clear_bit(tag - 1, &engine->context_tag);  in __execlists_schedule_in()
1362 		ce->lrc.ccid = tag << (GEN11_SW_CTX_ID_SHIFT - 32);  in __execlists_schedule_in()
1367 	ce->lrc.ccid |= engine->execlists.ccid;  in __execlists_schedule_in()
1369 	__intel_gt_pm_get(engine->gt);  in __execlists_schedule_in()
1370 	if (engine->fw_domain && !atomic_fetch_inc(&engine->fw_active))  in __execlists_schedule_in()
1371 		intel_uncore_forcewake_get(engine->uncore, engine->fw_domain);  in __execlists_schedule_in()
1381 	struct intel_context * const ce = rq->context;  in execlists_schedule_in()
1384 	GEM_BUG_ON(!intel_engine_pm_is_awake(rq->engine));  in execlists_schedule_in()
1387 	old = READ_ONCE(ce->inflight);  in execlists_schedule_in()
1390 			WRITE_ONCE(ce->inflight, __execlists_schedule_in(rq));  in execlists_schedule_in()
1393 	} while (!try_cmpxchg(&ce->inflight, &old, ptr_inc(old)));  in execlists_schedule_in()
1395 	GEM_BUG_ON(intel_context_inflight(ce) != rq->engine);  in execlists_schedule_in()
1402 	struct i915_request *next = READ_ONCE(ve->request);  in kick_siblings()
1404 	if (next == rq || (next && next->execution_mask & ~rq->execution_mask))  in kick_siblings()
1405 		tasklet_hi_schedule(&ve->base.execlists.tasklet);  in kick_siblings()
1413 	struct intel_context * const ce = rq->context;  in __execlists_schedule_out()
1416 	 * NB process_csb() is not under the engine->active.lock and hence  in __execlists_schedule_out()
1418 	 * refrain from doing non-trivial work here.  in __execlists_schedule_out()
1423 	 * idle and we want to re-enter powersaving.  in __execlists_schedule_out()
1425 	if (list_is_last_rcu(&rq->link, &ce->timeline->requests) &&  in __execlists_schedule_out()
1427 		intel_engine_add_retire(engine, ce->timeline);  in __execlists_schedule_out()
1429 	ccid >>= GEN11_SW_CTX_ID_SHIFT - 32;  in __execlists_schedule_out()
1433 		GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag));  in __execlists_schedule_out()
1434 		set_bit(ccid - 1, &engine->context_tag);  in __execlists_schedule_out()
1440 	if (engine->fw_domain && !atomic_dec_return(&engine->fw_active))  in __execlists_schedule_out()
1441 		intel_uncore_forcewake_put(engine->uncore, engine->fw_domain);  in __execlists_schedule_out()
1442 	intel_gt_pm_put_async(engine->gt);  in __execlists_schedule_out()
1453 	if (ce->engine != engine)  in __execlists_schedule_out()
1462 	struct intel_context * const ce = rq->context;  in execlists_schedule_out()
1468 	ccid = rq->context->lrc.ccid;  in execlists_schedule_out()
1469 	old = READ_ONCE(ce->inflight);  in execlists_schedule_out()
1472 	while (!try_cmpxchg(&ce->inflight, &old, cur));  in execlists_schedule_out()
1481 	struct intel_context *ce = rq->context;  in execlists_update_context()
1482 	u64 desc = ce->lrc.desc;  in execlists_update_context()
1498 	 * lite-restore and force it to reload the RING_TAIL. Otherwise, the  in execlists_update_context()
1502 	GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);  in execlists_update_context()
1503 	prev = rq->ring->tail;  in execlists_update_context()
1504 	tail = intel_ring_set_tail(rq->ring, rq->tail);  in execlists_update_context()
1505 	if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))  in execlists_update_context()
1507 	ce->lrc_reg_state[CTX_RING_TAIL] = tail;  in execlists_update_context()
1508 	rq->tail = rq->wa_tail;  in execlists_update_context()
1522 	ce->lrc.desc &= ~CTX_DESC_FORCE_RESTORE;  in execlists_update_context()
1528 	if (execlists->ctrl_reg) {  in write_desc()
1529 		writel(lower_32_bits(desc), execlists->submit_reg + port * 2);  in write_desc()
1530 		writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);  in write_desc()
1532 		writel(upper_32_bits(desc), execlists->submit_reg);  in write_desc()
1533 		writel(lower_32_bits(desc), execlists->submit_reg);  in write_desc()
1543 	snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",  in dump_port()
1545 		 rq->context->lrc.ccid,  in dump_port()
1546 		 rq->fence.context, rq->fence.seqno,  in dump_port()
1575 	return unlikely(!__tasklet_is_enabled(&execlists->tasklet));  in reset_in_progress()
1587 	u32 ccid = -1;  in assert_pending_valid()
1589 	trace_ports(execlists, msg, execlists->pending);  in assert_pending_valid()
1595 	if (!execlists->pending[0]) {  in assert_pending_valid()
1597 			      engine->name);  in assert_pending_valid()
1601 	if (execlists->pending[execlists_num_ports(execlists)]) {  in assert_pending_valid()
1603 			      engine->name, execlists_num_ports(execlists));  in assert_pending_valid()
1607 	for (port = execlists->pending; (rq = *port); port++) {  in assert_pending_valid()
1611 		GEM_BUG_ON(!kref_read(&rq->fence.refcount));  in assert_pending_valid()
1614 		if (ce == rq->context) {  in assert_pending_valid()
1616 				      engine->name,  in assert_pending_valid()
1617 				      ce->timeline->fence_context,  in assert_pending_valid()
1618 				      port - execlists->pending);  in assert_pending_valid()
1621 		ce = rq->context;  in assert_pending_valid()
1623 		if (ccid == ce->lrc.ccid) {  in assert_pending_valid()
1624 			GEM_TRACE_ERR("%s: Dup ccid:%x context:%llx in pending[%zd]\n",  in assert_pending_valid()
1625 				      engine->name,  in assert_pending_valid()
1626 				      ccid, ce->timeline->fence_context,  in assert_pending_valid()
1627 				      port - execlists->pending);  in assert_pending_valid()
1630 		ccid = ce->lrc.ccid;  in assert_pending_valid()
1639 				      engine->name,  in assert_pending_valid()
1640 				      ce->timeline->fence_context,  in assert_pending_valid()
1641 				      port - execlists->pending);  in assert_pending_valid()
1647 		if (!spin_trylock_irqsave(&rq->lock, flags))  in assert_pending_valid()
1653 		if (i915_active_is_idle(&ce->active) &&  in assert_pending_valid()
1656 				      engine->name,  in assert_pending_valid()
1657 				      ce->timeline->fence_context,  in assert_pending_valid()
1658 				      port - execlists->pending);  in assert_pending_valid()
1663 		if (!i915_vma_is_pinned(ce->state)) {  in assert_pending_valid()
1665 				      engine->name,  in assert_pending_valid()
1666 				      ce->timeline->fence_context,  in assert_pending_valid()
1667 				      port - execlists->pending);  in assert_pending_valid()
1672 		if (!i915_vma_is_pinned(ce->ring->vma)) {  in assert_pending_valid()
1674 				      engine->name,  in assert_pending_valid()
1675 				      ce->timeline->fence_context,  in assert_pending_valid()
1676 				      port - execlists->pending);  in assert_pending_valid()
1682 		spin_unlock_irqrestore(&rq->lock, flags);  in assert_pending_valid()
1692 	struct intel_engine_execlists *execlists = &engine->execlists;  in execlists_submit_ports()
1713 	for (n = execlists_num_ports(execlists); n--; ) {  in execlists_submit_ports()
1714 		struct i915_request *rq = execlists->pending[n];  in execlists_submit_ports()
1722 	if (execlists->ctrl_reg)  in execlists_submit_ports()
1723 		writel(EL_CTRL_LOAD, execlists->ctrl_reg);  in execlists_submit_ports()
1746 	return READ_ONCE(rq->fence.flags);  in i915_request_flags()
1760 	 * contexts, despite the best efforts of preempt-to-busy to confuse  in can_merge_rq()
1771 	if (!can_merge_ctx(prev->context, next->context))  in can_merge_rq()
1774 	GEM_BUG_ON(i915_seqno_passed(prev->fence.seqno, next->fence.seqno));  in can_merge_rq()
1790 	if (!(rq->execution_mask & engine->mask)) /* We peeked too soon! */  in virtual_matches()
1795 	 * (i.e. when we have seen the final CS event switching out of  in virtual_matches()
1802 	inflight = intel_context_inflight(&ve->context);  in virtual_matches()
1814 	if (likely(engine == ve->siblings[0]))  in virtual_xfer_context()
1817 	GEM_BUG_ON(READ_ONCE(ve->context.inflight));  in virtual_xfer_context()
1819 		virtual_update_register_offsets(ve->context.lrc_reg_state,  in virtual_xfer_context()
1828 	for (n = 1; n < ve->num_siblings; n++) {  in virtual_xfer_context()
1829 		if (ve->siblings[n] == engine) {  in virtual_xfer_context()
1830 			swap(ve->siblings[n], ve->siblings[0]);  in virtual_xfer_context()
1838 				     &(rq__)->sched.waiters_list, \
1843 				&(rq__)->sched.signalers_list, \
1852 	 * the round-robin list (i.e. its priority level), but  in defer_request()
1861 		list_move_tail(&rq->sched.link, pl);  in defer_request()
1865 				container_of(p->waiter, typeof(*w), sched);  in defer_request()
1867 			if (p->flags & I915_DEPENDENCY_WEAK)  in defer_request()
1871 			if (w->engine != rq->engine)  in defer_request()
1887 			list_move_tail(&w->sched.link, &list);  in defer_request()
1915 	hint = engine->execlists.queue_priority_hint;  in need_timeslice()
1919 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in need_timeslice()
1921 			intel_context_inflight(&ve->context);  in need_timeslice()
1927 			next = READ_ONCE(ve->request);  in need_timeslice()
1934 	if (!list_is_last(&rq->sched.link, &engine->active.requests))  in need_timeslice()
1948 	 * If the active context ever busy-waited on a semaphore,  in timeslice_yield()
1951 	 * possibly even its own lite-restore). The HW only sends an interrupt  in timeslice_yield()
1954 	 * safe, yield if it might be stuck -- it will be given a fresh  in timeslice_yield()
1957 	return rq->context->lrc.ccid == READ_ONCE(el->yield);  in timeslice_yield()
1964 	return timer_expired(&el->timer) || timeslice_yield(el, rq);  in timeslice_expired()
1970 	if (list_is_last(&rq->sched.link, &engine->active.requests))  in switch_prio()
1971 		return engine->execlists.queue_priority_hint;  in switch_prio()
1979 	return READ_ONCE(engine->props.timeslice_duration_ms);  in timeslice()
1984 	const struct intel_engine_execlists *execlists = &engine->execlists;  in active_timeslice()
1985 	const struct i915_request *rq = *execlists->active;  in active_timeslice()
1990 	if (READ_ONCE(execlists->switch_priority_hint) < effective_prio(rq))  in active_timeslice()
2006 	set_timer_ms(&engine->execlists.timer, duration);  in set_timeslice()
2011 	struct intel_engine_execlists *execlists = &engine->execlists;  in start_timeslice()
2017 	WRITE_ONCE(execlists->switch_priority_hint, prio);  in start_timeslice()
2021 	if (timer_pending(&execlists->timer))  in start_timeslice()
2029 	set_timer_ms(&execlists->timer, duration);  in start_timeslice()
2034 	(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);  in record_preemption()
2044 	if (unlikely(intel_context_is_banned(rq->context)))  in active_preempt_timeout()
2047 	return READ_ONCE(engine->props.preempt_timeout_ms);  in active_preempt_timeout()
2056 	set_timer_ms(&engine->execlists.preempt,  in set_preempt_timeout()
2069 	while (count--)  in copy_ports()
2075 	struct intel_engine_execlists * const execlists = &engine->execlists;  in execlists_dequeue()
2076 	struct i915_request **port = execlists->pending;  in execlists_dequeue()
2077 	struct i915_request ** const last_port = port + execlists->port_mask;  in execlists_dequeue()
2088 	 * is maintained by the CS in the context image, it marks the place  in execlists_dequeue()
2089 	 * where it got up to last time, and through RING_TAIL we tell the CS  in execlists_dequeue()
2097 	 *                                    ^- RING_TAIL  in execlists_dequeue()
2098 	 * since to execute req2 the CS must first execute req1.  in execlists_dequeue()
2105 	for (rb = rb_first_cached(&execlists->virtual); rb; ) {  in execlists_dequeue()
2107 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in execlists_dequeue()
2108 		struct i915_request *rq = READ_ONCE(ve->request);  in execlists_dequeue()
2111 			rb_erase_cached(rb, &execlists->virtual);  in execlists_dequeue()
2113 			rb = rb_first_cached(&execlists->virtual);  in execlists_dequeue()
2133 	active = READ_ONCE(execlists->active);  in execlists_dequeue()
2150 				tasklet_hi_schedule(&execlists->tasklet);  in execlists_dequeue()
2156 				     last->fence.context,  in execlists_dequeue()
2157 				     last->fence.seqno,  in execlists_dequeue()
2158 				     last->sched.attr.priority,  in execlists_dequeue()
2159 				     execlists->queue_priority_hint);  in execlists_dequeue()
2182 				tasklet_hi_schedule(&execlists->tasklet);  in execlists_dequeue()
2188 				     last->fence.context,  in execlists_dequeue()
2189 				     last->fence.seqno,  in execlists_dequeue()
2190 				     last->sched.attr.priority,  in execlists_dequeue()
2191 				     execlists->queue_priority_hint,  in execlists_dequeue()
2202 			 * force a full context restore, as a lite-restore  in execlists_dequeue()
2215 			 * just wait until the next CS event before  in execlists_dequeue()
2217 			 * lite-restore preemption event, but if we wait  in execlists_dequeue()
2221 			if (!list_is_last(&last->sched.link,  in execlists_dequeue()
2222 					  &engine->active.requests)) {  in execlists_dequeue()
2235 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in execlists_dequeue()
2238 		spin_lock(&ve->base.active.lock);  in execlists_dequeue()
2240 		rq = ve->request;  in execlists_dequeue()
2242 			spin_unlock(&ve->base.active.lock);  in execlists_dequeue()
2243 			rb_erase_cached(rb, &execlists->virtual);  in execlists_dequeue()
2245 			rb = rb_first_cached(&execlists->virtual);  in execlists_dequeue()
2249 		GEM_BUG_ON(rq != ve->request);  in execlists_dequeue()
2250 		GEM_BUG_ON(rq->engine != &ve->base);  in execlists_dequeue()
2251 		GEM_BUG_ON(rq->context != &ve->context);  in execlists_dequeue()
2255 				spin_unlock(&ve->base.active.lock);  in execlists_dequeue()
2261 				spin_unlock(&ve->base.active.lock);  in execlists_dequeue()
2268 				     rq->fence.context,  in execlists_dequeue()
2269 				     rq->fence.seqno,  in execlists_dequeue()
2273 				     yesno(engine != ve->siblings[0]));  in execlists_dequeue()
2275 			WRITE_ONCE(ve->request, NULL);  in execlists_dequeue()
2276 			WRITE_ONCE(ve->base.execlists.queue_priority_hint,  in execlists_dequeue()
2278 			rb_erase_cached(rb, &execlists->virtual);  in execlists_dequeue()
2281 			GEM_BUG_ON(!(rq->execution_mask & engine->mask));  in execlists_dequeue()
2282 			WRITE_ONCE(rq->engine, engine);  in execlists_dequeue()
2294 				 * ve->siblings[] on an idle context, where  in execlists_dequeue()
2295 				 * we may be using ve->siblings[] in  in execlists_dequeue()
2299 				GEM_BUG_ON(ve->siblings[0] != engine);  in execlists_dequeue()
2309 			 * preempt-to-busy!). Keep looking at the veng queue  in execlists_dequeue()
2314 				spin_unlock(&ve->base.active.lock);  in execlists_dequeue()
2315 				rb = rb_first_cached(&execlists->virtual);  in execlists_dequeue()
2320 		spin_unlock(&ve->base.active.lock);  in execlists_dequeue()
2324 	while ((rb = rb_first_cached(&execlists->queue))) {  in execlists_dequeue()
2357 				if (last->context == rq->context)  in execlists_dequeue()
2370 				if (ctx_single_port_submission(last->context) ||  in execlists_dequeue()
2371 				    ctx_single_port_submission(rq->context))  in execlists_dequeue()
2379 					*port = execlists_schedule_in(last, port - execlists->pending);  in execlists_dequeue()
2385 					   !can_merge_ctx(last->context,  in execlists_dequeue()
2386 							  rq->context));  in execlists_dequeue()
2388 					   i915_seqno_passed(last->fence.seqno,  in execlists_dequeue()
2389 							     rq->fence.seqno));  in execlists_dequeue()
2396 		rb_erase_cached(&p->node, &execlists->queue);  in execlists_dequeue()
2402 	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.  in execlists_dequeue()
2407 	 * also be prepared to reorder requests as they are in-flight on the  in execlists_dequeue()
2417 	execlists->queue_priority_hint = queue_prio(execlists);  in execlists_dequeue()
2420 		*port = execlists_schedule_in(last, port - execlists->pending);  in execlists_dequeue()
2421 		execlists->switch_priority_hint =  in execlists_dequeue()
2422 			switch_prio(engine, *execlists->pending);  in execlists_dequeue()
2428 		if (!memcmp(active, execlists->pending,  in execlists_dequeue()
2429 			    (port - execlists->pending + 1) * sizeof(*port))) {  in execlists_dequeue()
2432 			while (port-- != execlists->pending);  in execlists_dequeue()
2436 		clear_ports(port + 1, last_port - port);  in execlists_dequeue()
2438 		WRITE_ONCE(execlists->yield, -1);  in execlists_dequeue()
2442 		start_timeslice(engine, execlists->queue_priority_hint);  in execlists_dequeue()
2453 	for (port = execlists->pending; *port; port++)  in cancel_port_requests()
2455 	clear_ports(execlists->pending, ARRAY_SIZE(execlists->pending));  in cancel_port_requests()
2458 	for (port = xchg(&execlists->active, execlists->pending); *port; port++)  in cancel_port_requests()
2460 	clear_ports(execlists->inflight, ARRAY_SIZE(execlists->inflight));  in cancel_port_requests()
2463 	WRITE_ONCE(execlists->active, execlists->inflight);  in cancel_port_requests()
2480  *     bits 3-5:   engine class
2481  *     bits 6-11:  engine instance
2482  *     bits 12-14: reserved
2483  *     bits 15-25: sw context id of the lrc the GT switched to
2484  *     bits 26-31: sw counter of the lrc the GT switched to
2485  *     bits 32-35: context switch detail
2486  *                  - 0: ctx complete
2487  *                  - 1: wait on sync flip
2488  *                  - 2: wait on vblank
2489  *                  - 3: wait on scanline
2490  *                  - 4: wait on semaphore
2491  *                  - 5: context preempted (not on SEMAPHORE_WAIT or
2494  *     bits 37-43: wait detail (for switch detail 1 to 4)
2495  *     bits 44-46: reserved
2496  *     bits 47-57: sw context id of the lrc the GT switched away from
2497  *     bits 58-63: sw counter of the lrc the GT switched away from
2507 	if (unlikely(entry == -1)) {  in gen12_csb_parse()
2509 		if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 50))  in gen12_csb_parse()
2513 	WRITE_ONCE(*(u64 *)csb, -1);  in gen12_csb_parse()
2523 	 * instructions and lite-restore. Preempt-to-idle via the CTRL register  in gen12_csb_parse()
2547 	struct intel_engine_execlists * const execlists = &engine->execlists;  in process_csb()
2548 	const u64 * const buf = execlists->csb_status;  in process_csb()
2549 	const u8 num_entries = execlists->csb_size;  in process_csb()
2557 	GEM_BUG_ON(!tasklet_is_locked(&execlists->tasklet) &&  in process_csb()
2571 	head = execlists->csb_head;  in process_csb()
2572 	tail = READ_ONCE(*execlists->csb_write);  in process_csb()
2592 	execlists->csb_head = tail;  in process_csb()
2593 	ENGINE_TRACE(engine, "cs-irq head=%d, tail=%d\n", head, tail);  in process_csb()
2619 		 * context-switch). As we only hold the reference to the  in process_csb()
2621 		 * is subject to a potential use-after-free. Thus we  in process_csb()
2628 		ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n",  in process_csb()
2633 		if (INTEL_GEN(engine->i915) >= 12)  in process_csb()
2638 			struct i915_request * const *old = execlists->active;  in process_csb()
2640 			if (GEM_WARN_ON(!*execlists->pending)) {  in process_csb()
2641 				execlists->error_interrupt |= ERROR_CSB;  in process_csb()
2648 			WRITE_ONCE(execlists->active, execlists->pending);  in process_csb()
2658 			copy_ports(execlists->inflight,  in process_csb()
2659 				   execlists->pending,  in process_csb()
2662 			WRITE_ONCE(execlists->active, execlists->inflight);  in process_csb()
2667 			WRITE_ONCE(execlists->pending[0], NULL);  in process_csb()
2669 			if (GEM_WARN_ON(!*execlists->active)) {  in process_csb()
2670 				execlists->error_interrupt |= ERROR_CSB;  in process_csb()
2675 			trace_ports(execlists, "completed", execlists->active);  in process_csb()
2683 			 * user interrupt and the CS event for the context  in process_csb()
2684 			 * switch would therefore be before the CS event  in process_csb()
2688 			    !i915_request_completed(*execlists->active)) {  in process_csb()
2689 				struct i915_request *rq = *execlists->active;  in process_csb()
2691 					rq->context->lrc_reg_state;  in process_csb()
2696 					     "ring:{start:0x%08x, head:%04x, tail:%04x, ctl:%08x, mode:%08x}\n",  in process_csb()
2703 					     "rq:{start:%08x, head:%04x, tail:%04x, seqno:%llx:%d, hwsp:%d}, ",  in process_csb()
2704 					     i915_ggtt_offset(rq->ring->vma),  in process_csb()
2705 					     rq->head, rq->tail,  in process_csb()
2706 					     rq->fence.context,  in process_csb()
2707 					     lower_32_bits(rq->fence.seqno),  in process_csb()
2710 					     "ctx:{start:%08x, head:%04x, tail:%04x}, ",  in process_csb()
2716 			execlists_schedule_out(*execlists->active++);  in process_csb()
2718 			GEM_BUG_ON(execlists->active - execlists->inflight >  in process_csb()
2736 	invalidate_csb_entries(&buf[0], &buf[num_entries - 1]);  in process_csb()
2741 	lockdep_assert_held(&engine->active.lock);  in __execlists_submission_tasklet()
2742 	if (!READ_ONCE(engine->execlists.pending[0])) {  in __execlists_submission_tasklet()
2743 		rcu_read_lock(); /* protect peeking at execlists->active */  in __execlists_submission_tasklet()
2759 		clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);  in __execlists_hold()
2760 		list_move_tail(&rq->sched.link, &rq->engine->active.hold);  in __execlists_hold()
2766 				container_of(p->waiter, typeof(*w), sched);  in __execlists_hold()
2769 			if (w->engine != rq->engine)  in __execlists_hold()
2781 			list_move_tail(&w->sched.link, &list);  in __execlists_hold()
2794 	spin_lock_irq(&engine->active.lock);  in execlists_hold()
2801 	if (rq->engine != engine) { /* preempted virtual engine */  in execlists_hold()
2802 		struct virtual_engine *ve = to_virtual_engine(rq->engine);  in execlists_hold()
2811 		GEM_BUG_ON(!reset_in_progress(&engine->execlists));  in execlists_hold()
2820 		spin_lock(&ve->base.active.lock);  in execlists_hold()
2821 		GEM_BUG_ON(intel_context_inflight(rq->context) != engine);  in execlists_hold()
2822 		GEM_BUG_ON(ve->request != rq);  in execlists_hold()
2823 		ve->request = NULL;  in execlists_hold()
2824 		spin_unlock(&ve->base.active.lock);  in execlists_hold()
2827 		rq->engine = engine;  in execlists_hold()
2837 	GEM_BUG_ON(rq->engine != engine);  in execlists_hold()
2839 	GEM_BUG_ON(list_empty(&engine->active.hold));  in execlists_hold()
2842 	spin_unlock_irq(&engine->active.lock);  in execlists_hold()
2858 			container_of(p->signaler, typeof(*s), sched);  in hold_request()
2860 		if (s->engine != rq->engine)  in hold_request()
2882 		GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));  in __execlists_unhold()
2885 		list_move_tail(&rq->sched.link,  in __execlists_unhold()
2886 			       i915_sched_lookup_priolist(rq->engine,  in __execlists_unhold()
2888 		set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);  in __execlists_unhold()
2893 				container_of(p->waiter, typeof(*w), sched);  in __execlists_unhold()
2896 			if (rq->fence.error)  in __execlists_unhold()
2897 				i915_request_set_error_once(w, rq->fence.error);  in __execlists_unhold()
2899 			if (w->engine != rq->engine)  in __execlists_unhold()
2909 			list_move_tail(&w->sched.link, &list);  in __execlists_unhold()
2919 	spin_lock_irq(&engine->active.lock);  in execlists_unhold()
2927 	if (rq_prio(rq) > engine->execlists.queue_priority_hint) {  in execlists_unhold()
2928 		engine->execlists.queue_priority_hint = rq_prio(rq);  in execlists_unhold()
2929 		tasklet_hi_schedule(&engine->execlists.tasklet);  in execlists_unhold()
2932 	spin_unlock_irq(&engine->active.lock);  in execlists_unhold()
2945 	struct intel_engine_cs *engine = cap->rq->engine;  in execlists_capture_work()
2946 	struct intel_gt_coredump *gt = cap->error->gt;  in execlists_capture_work()
2950 	vma = intel_engine_coredump_add_request(gt->engine, cap->rq, gfp);  in execlists_capture_work()
2955 		intel_engine_coredump_add_vma(gt->engine, vma, compress);  in execlists_capture_work()
2959 	gt->simulated = gt->engine->simulated;  in execlists_capture_work()
2960 	cap->error->simulated = gt->simulated;  in execlists_capture_work()
2963 	i915_error_state_store(cap->error);  in execlists_capture_work()
2964 	i915_gpu_coredump_put(cap->error);  in execlists_capture_work()
2967 	execlists_unhold(engine, cap->rq);  in execlists_capture_work()
2968 	i915_request_put(cap->rq);  in execlists_capture_work()
2982 	cap->error = i915_gpu_coredump_alloc(engine->i915, gfp);  in capture_regs()
2983 	if (!cap->error)  in capture_regs()
2986 	cap->error->gt = intel_gt_coredump_alloc(engine->gt, gfp);  in capture_regs()
2987 	if (!cap->error->gt)  in capture_regs()
2990 	cap->error->gt->engine = intel_engine_coredump_alloc(engine, gfp);  in capture_regs()
2991 	if (!cap->error->gt->engine)  in capture_regs()
2997 	kfree(cap->error->gt);  in capture_regs()
2999 	kfree(cap->error);  in capture_regs()
3008 	const struct intel_engine_execlists * const el = &engine->execlists;  in active_context()
3013 	 * we trigger an error (via interrupt) before the first CS event has  in active_context()
3017 	for (port = el->active; (rq = *port); port++) {  in active_context()
3018 		if (rq->context->lrc.ccid == ccid) {  in active_context()
3021 				     port - el->active);  in active_context()
3026 	for (port = el->pending; (rq = *port); port++) {  in active_context()
3027 		if (rq->context->lrc.ccid == ccid) {  in active_context()
3030 				     port - el->pending);  in active_context()
3035 	ENGINE_TRACE(engine, "ccid:%x not found\n", ccid);  in active_context()
3060 	spin_lock_irq(&engine->active.lock);  in execlists_capture()
3061 	cap->rq = active_context(engine, active_ccid(engine));  in execlists_capture()
3062 	if (cap->rq) {  in execlists_capture()
3063 		cap->rq = active_request(cap->rq->context->timeline, cap->rq);  in execlists_capture()
3064 		cap->rq = i915_request_get_rcu(cap->rq);  in execlists_capture()
3066 	spin_unlock_irq(&engine->active.lock);  in execlists_capture()
3067 	if (!cap->rq)  in execlists_capture()
3086 	 * fast enough to run inside an irq-off atomic section!), so we will  in execlists_capture()
3087 	 * simply hold that request accountable for being non-preemptible  in execlists_capture()
3090 	if (!execlists_hold(engine, cap->rq))  in execlists_capture()
3093 	INIT_WORK(&cap->work, execlists_capture_work);  in execlists_capture()
3094 	schedule_work(&cap->work);  in execlists_capture()
3098 	i915_request_put(cap->rq);  in execlists_capture()
3100 	i915_gpu_coredump_put(cap->error);  in execlists_capture()
3106 	const unsigned int bit = I915_RESET_ENGINE + engine->id;  in execlists_reset()
3107 	unsigned long *lock = &engine->gt->reset.flags;  in execlists_reset()
3109 	if (!intel_has_reset_engine(engine->gt))  in execlists_reset()
3118 	tasklet_disable_nosync(&engine->execlists.tasklet);  in execlists_reset()
3124 	tasklet_enable(&engine->execlists.tasklet);  in execlists_reset()
3130 	const struct timer_list *t = &engine->execlists.preempt;  in preempt_timeout()
3138 	return READ_ONCE(engine->execlists.pending[0]);  in preempt_timeout()
3152 	if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) {  in execlists_submission_tasklet()
3156 		if (engine->execlists.error_interrupt & GENMASK(15, 0))  in execlists_submission_tasklet()
3157 			msg = "CS error"; /* thrown by a user payload */  in execlists_submission_tasklet()
3158 		else if (engine->execlists.error_interrupt & ERROR_CSB)  in execlists_submission_tasklet()
3163 		engine->execlists.error_interrupt = 0;  in execlists_submission_tasklet()
3167 	if (!READ_ONCE(engine->execlists.pending[0]) || timeout) {  in execlists_submission_tasklet()
3170 		spin_lock_irqsave(&engine->active.lock, flags);  in execlists_submission_tasklet()
3172 		spin_unlock_irqrestore(&engine->active.lock, flags);  in execlists_submission_tasklet()
3174 		/* Recheck after serialising with direct-submission */  in execlists_submission_tasklet()
3176 			cancel_timer(&engine->execlists.preempt);  in execlists_submission_tasklet()
3185 	tasklet_hi_schedule(&execlists->tasklet);  in __execlists_kick()
3204 	GEM_BUG_ON(!list_empty(&rq->sched.link));  in queue_request()
3205 	list_add_tail(&rq->sched.link,  in queue_request()
3207 	set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);  in queue_request()
3212 	struct intel_engine_execlists * const execlists = &engine->execlists;  in __submit_queue_imm()
3223 	struct intel_engine_execlists *execlists = &engine->execlists;  in submit_queue()
3225 	if (rq_prio(rq) <= execlists->queue_priority_hint)  in submit_queue()
3228 	execlists->queue_priority_hint = rq_prio(rq);  in submit_queue()
3236 	return !list_empty(&engine->active.hold) && hold_request(rq);  in ancestor_on_hold()
3241 	struct intel_engine_execlists *el = &engine->execlists;  in flush_csb()
3243 	if (READ_ONCE(el->pending[0]) && tasklet_trylock(&el->tasklet)) {  in flush_csb()
3246 		tasklet_unlock(&el->tasklet);  in flush_csb()
3252 	struct intel_engine_cs *engine = request->engine;  in execlists_submit_request()
3255 	/* Hopefully we clear execlists->pending[] to let us through */  in execlists_submit_request()
3258 	/* Will be called from irq-context when using foreign fences. */  in execlists_submit_request()
3259 	spin_lock_irqsave(&engine->active.lock, flags);  in execlists_submit_request()
3263 		list_add_tail(&request->sched.link, &engine->active.hold);  in execlists_submit_request()
3268 		GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));  in execlists_submit_request()
3269 		GEM_BUG_ON(list_empty(&request->sched.link));  in execlists_submit_request()
3274 	spin_unlock_irqrestore(&engine->active.lock, flags);  in execlists_submit_request()
3279 	intel_ring_put(ce->ring);  in __execlists_context_fini()
3280 	i915_vma_put(ce->state);  in __execlists_context_fini()
3287 	GEM_BUG_ON(!i915_active_is_idle(&ce->active));  in execlists_context_destroy()
3290 	if (ce->state)  in execlists_context_destroy()
3303 	vaddr += engine->context_size;  in set_redzone()
3314 	vaddr += engine->context_size;  in check_redzone()
3317 		drm_err_once(&engine->i915->drm,  in check_redzone()
3319 			     engine->name);  in check_redzone()
3324 	check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET,  in execlists_context_unpin()
3325 		      ce->engine);  in execlists_context_unpin()
3330 	i915_gem_object_unpin_map(ce->state->obj);  in execlists_context_post_unpin()
3334 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs)  in gen12_emit_timestamp_wa()  argument
3336 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in gen12_emit_timestamp_wa()
3339 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_timestamp_wa()
3340 	*cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET +  in gen12_emit_timestamp_wa()
3342 	*cs++ = 0;  in gen12_emit_timestamp_wa()
3344 	*cs++ = MI_LOAD_REGISTER_REG |  in gen12_emit_timestamp_wa()
3347 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_timestamp_wa()
3348 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0));  in gen12_emit_timestamp_wa()
3350 	*cs++ = MI_LOAD_REGISTER_REG |  in gen12_emit_timestamp_wa()
3353 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_timestamp_wa()
3354 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0));  in gen12_emit_timestamp_wa()
3356 	return cs;  in gen12_emit_timestamp_wa()
3360 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs)  in gen12_emit_restore_scratch()  argument
3362 	GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1);  in gen12_emit_restore_scratch()
3364 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in gen12_emit_restore_scratch()
3367 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_restore_scratch()
3368 	*cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET +  in gen12_emit_restore_scratch()
3369 		(lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32);  in gen12_emit_restore_scratch()
3370 	*cs++ = 0;  in gen12_emit_restore_scratch()
3372 	return cs;  in gen12_emit_restore_scratch()
3376 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs)  in gen12_emit_cmd_buf_wa()  argument
3378 	GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1);  in gen12_emit_cmd_buf_wa()
3380 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in gen12_emit_cmd_buf_wa()
3383 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_cmd_buf_wa()
3384 	*cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET +  in gen12_emit_cmd_buf_wa()
3385 		(lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32);  in gen12_emit_cmd_buf_wa()
3386 	*cs++ = 0;  in gen12_emit_cmd_buf_wa()
3388 	*cs++ = MI_LOAD_REGISTER_REG |  in gen12_emit_cmd_buf_wa()
3391 	*cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0));  in gen12_emit_cmd_buf_wa()
3392 	*cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0));  in gen12_emit_cmd_buf_wa()
3394 	return cs;  in gen12_emit_cmd_buf_wa()
3398 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs)  in gen12_emit_indirect_ctx_rcs()  argument
3400 	cs = gen12_emit_timestamp_wa(ce, cs);  in gen12_emit_indirect_ctx_rcs()
3401 	cs = gen12_emit_cmd_buf_wa(ce, cs);  in gen12_emit_indirect_ctx_rcs()
3402 	cs = gen12_emit_restore_scratch(ce, cs);  in gen12_emit_indirect_ctx_rcs()
3404 	return cs;  in gen12_emit_indirect_ctx_rcs()
3408 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs)  in gen12_emit_indirect_ctx_xcs()  argument
3410 	cs = gen12_emit_timestamp_wa(ce, cs);  in gen12_emit_indirect_ctx_xcs()
3411 	cs = gen12_emit_restore_scratch(ce, cs);  in gen12_emit_indirect_ctx_xcs()
3413 	return cs;  in gen12_emit_indirect_ctx_xcs()
3418 	return PAGE_SIZE * ce->wa_bb_page;  in context_wa_bb_offset()
3425 	GEM_BUG_ON(!ce->wa_bb_page);  in context_indirect_bb()
3427 	ptr = ce->lrc_reg_state;  in context_indirect_bb()
3428 	ptr -= LRC_STATE_OFFSET; /* back to start of context image */  in context_indirect_bb()
3440 	u32 *cs;  in setup_indirect_ctx_bb()  local
3442 	cs = emit(ce, start);  in setup_indirect_ctx_bb()
3443 	GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs));  in setup_indirect_ctx_bb()
3444 	while ((unsigned long)cs % CACHELINE_BYTES)  in setup_indirect_ctx_bb()
3445 		*cs++ = MI_NOOP;  in setup_indirect_ctx_bb()
3447 	lrc_ring_setup_indirect_ctx(ce->lrc_reg_state, engine,  in setup_indirect_ctx_bb()
3448 				    i915_ggtt_offset(ce->state) +  in setup_indirect_ctx_bb()
3450 				    (cs - start) * sizeof(*cs));  in setup_indirect_ctx_bb()
3458 	struct intel_ring *ring = ce->ring;  in __execlists_update_reg_state()
3459 	u32 *regs = ce->lrc_reg_state;  in __execlists_update_reg_state()
3462 	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));  in __execlists_update_reg_state()
3464 	regs[CTX_RING_START] = i915_ggtt_offset(ring->vma);  in __execlists_update_reg_state()
3466 	regs[CTX_RING_TAIL] = ring->tail;  in __execlists_update_reg_state()
3467 	regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID;  in __execlists_update_reg_state()
3470 	if (engine->class == RENDER_CLASS) {  in __execlists_update_reg_state()
3472 			intel_sseu_make_rpcs(engine->gt, &ce->sseu);  in __execlists_update_reg_state()
3477 	if (ce->wa_bb_page) {  in __execlists_update_reg_state()
3478 		u32 *(*fn)(const struct intel_context *ce, u32 *cs);  in __execlists_update_reg_state()
3481 		if (ce->engine->class == RENDER_CLASS)  in __execlists_update_reg_state()
3485 		GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size);  in __execlists_update_reg_state()
3494 	GEM_BUG_ON(!ce->state);  in execlists_context_pre_pin()
3495 	GEM_BUG_ON(!i915_vma_is_pinned(ce->state));  in execlists_context_pre_pin()
3497 	*vaddr = i915_gem_object_pin_map(ce->state->obj,  in execlists_context_pre_pin()
3498 					i915_coherent_map_type(ce->engine->i915) |  in execlists_context_pre_pin()
3509 	ce->lrc.lrca = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE;  in __execlists_context_pin()
3510 	ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET;  in __execlists_context_pin()
3511 	__execlists_update_reg_state(ce, engine, ce->ring->tail);  in __execlists_context_pin()
3518 	return __execlists_context_pin(ce, ce->engine, vaddr);  in execlists_context_pin()
3523 	return __execlists_context_alloc(ce, ce->engine);  in execlists_context_alloc()
3531 	intel_ring_reset(ce->ring, ce->ring->emit);  in execlists_context_reset()
3534 	execlists_init_reg_state(ce->lrc_reg_state,  in execlists_context_reset()
3535 				 ce, ce->engine, ce->ring, true);  in execlists_context_reset()
3536 	__execlists_update_reg_state(ce, ce->engine, ce->ring->tail);  in execlists_context_reset()
3538 	ce->lrc.desc |= CTX_DESC_FORCE_RESTORE;  in execlists_context_reset()
3562 	cl = rcu_dereference_protected(rq->hwsp_cacheline, 1);  in hwsp_offset()
3564 		return cl->ggtt_offset;  in hwsp_offset()
3566 	return rcu_dereference_protected(rq->timeline, 1)->hwsp_offset;  in hwsp_offset()
3571 	u32 *cs;  in gen8_emit_init_breadcrumb()  local
3574 	if (!i915_request_timeline(rq)->has_initial_breadcrumb)  in gen8_emit_init_breadcrumb()
3577 	cs = intel_ring_begin(rq, 6);  in gen8_emit_init_breadcrumb()
3578 	if (IS_ERR(cs))  in gen8_emit_init_breadcrumb()
3579 		return PTR_ERR(cs);  in gen8_emit_init_breadcrumb()
3587 	*cs++ = MI_ARB_CHECK;  in gen8_emit_init_breadcrumb()
3588 	*cs++ = MI_NOOP;  in gen8_emit_init_breadcrumb()
3590 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in gen8_emit_init_breadcrumb()
3591 	*cs++ = hwsp_offset(rq);  in gen8_emit_init_breadcrumb()
3592 	*cs++ = 0;  in gen8_emit_init_breadcrumb()
3593 	*cs++ = rq->fence.seqno - 1;  in gen8_emit_init_breadcrumb()
3595 	intel_ring_advance(rq, cs);  in gen8_emit_init_breadcrumb()
3598 	rq->infix = intel_ring_offset(rq, cs);  in gen8_emit_init_breadcrumb()
3600 	__set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags);  in gen8_emit_init_breadcrumb()
3607 	const struct intel_engine_cs * const engine = rq->engine;  in emit_pdps()
3608 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->context->vm);  in emit_pdps()
3610 	u32 *cs;  in emit_pdps()  local
3612 	GEM_BUG_ON(intel_vgpu_active(rq->engine->i915));  in emit_pdps()
3622 	err = engine->emit_flush(rq, EMIT_FLUSH);  in emit_pdps()
3627 	err = engine->emit_flush(rq, EMIT_INVALIDATE);  in emit_pdps()
3631 	cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);  in emit_pdps()
3632 	if (IS_ERR(cs))  in emit_pdps()
3633 		return PTR_ERR(cs);  in emit_pdps()
3636 	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;  in emit_pdps()
3637 	for (i = GEN8_3LVL_PDPES; i--; ) {  in emit_pdps()
3639 		u32 base = engine->mmio_base;  in emit_pdps()
3641 		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));  in emit_pdps()
3642 		*cs++ = upper_32_bits(pd_daddr);  in emit_pdps()
3643 		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));  in emit_pdps()
3644 		*cs++ = lower_32_bits(pd_daddr);  in emit_pdps()
3646 	*cs++ = MI_NOOP;  in emit_pdps()
3648 	intel_ring_advance(rq, cs);  in emit_pdps()
3657 	GEM_BUG_ON(!intel_context_is_pinned(request->context));  in execlists_request_alloc()
3661 	 * we start building the request - in which case we will just  in execlists_request_alloc()
3664 	request->reserved_space += EXECLISTS_REQUEST_SIZE;  in execlists_request_alloc()
3674 	if (!i915_vm_is_4lvl(request->context->vm)) {  in execlists_request_alloc()
3681 	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);  in execlists_request_alloc()
3685 	request->reserved_space -= EXECLISTS_REQUEST_SIZE;  in execlists_request_alloc()
3698  * it for a short period and this batch in non-premptible. We can ofcourse
3711 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
3726 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
3754 	if (IS_BROADWELL(engine->i915))  in gen8_init_indirectctx_bb()
3792 		*batch++ = i915_mmio_reg_offset(lri->reg);  in emit_lri()
3793 		*batch++ = lri->value;  in emit_lri()
3794 	} while (lri++, --count);  in emit_lri()
3841 	if (HAS_POOLED_EU(engine->i915)) {  in gen9_init_indirectctx_bb()
3914 	obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_OBJ_SIZE);  in lrc_setup_wa_ctx()
3918 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in lrc_setup_wa_ctx()
3928 	engine->wa_ctx.vma = vma;  in lrc_setup_wa_ctx()
3938 	i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);  in lrc_destroy_wa_ctx()
3945 	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;  in intel_init_workaround_bb()
3946 	struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,  in intel_init_workaround_bb()
3947 					    &wa_ctx->per_ctx };  in intel_init_workaround_bb()
3953 	if (engine->class != RENDER_CLASS)  in intel_init_workaround_bb()
3956 	switch (INTEL_GEN(engine->i915)) {  in intel_init_workaround_bb()
3973 		MISSING_CASE(INTEL_GEN(engine->i915));  in intel_init_workaround_bb()
3979 		drm_dbg(&engine->i915->drm,  in intel_init_workaround_bb()
3984 	batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB);  in intel_init_workaround_bb()
3993 		wa_bb[i]->offset = batch_ptr - batch;  in intel_init_workaround_bb()
3994 		if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset,  in intel_init_workaround_bb()
3996 			ret = -EINVAL;  in intel_init_workaround_bb()
4001 		wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);  in intel_init_workaround_bb()
4003 	GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);  in intel_init_workaround_bb()
4005 	__i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch);  in intel_init_workaround_bb()
4006 	__i915_gem_object_release_map(wa_ctx->vma->obj);  in intel_init_workaround_bb()
4015 	struct intel_engine_execlists * const execlists = &engine->execlists;  in reset_csb_pointers()
4016 	const unsigned int reset_value = execlists->csb_size - 1;  in reset_csb_pointers()
4037 	execlists->csb_head = reset_value;  in reset_csb_pointers()
4038 	WRITE_ONCE(*execlists->csb_write, reset_value);  in reset_csb_pointers()
4042 	memset(execlists->csb_status, -1, (reset_value + 1) * sizeof(u64));  in reset_csb_pointers()
4043 	invalidate_csb_entries(&execlists->csb_status[0],  in reset_csb_pointers()
4044 			       &execlists->csb_status[reset_value]);  in reset_csb_pointers()
4051 	GEM_BUG_ON(READ_ONCE(*execlists->csb_write) != reset_value);  in reset_csb_pointers()
4066 		memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);  in execlists_sanitize()
4075 	intel_timeline_reset_seqno(engine->kernel_context->timeline);  in execlists_sanitize()
4078 	clflush_cache_range(engine->status_page.addr, PAGE_SIZE);  in execlists_sanitize()
4085 	engine->execlists.error_interrupt = 0;  in enable_error_interrupt()
4091 		drm_err(&engine->i915->drm,  in enable_error_interrupt()
4092 			"engine '%s' resumed still in error: %08x\n",  in enable_error_interrupt()
4093 			engine->name, status);  in enable_error_interrupt()
4094 		__intel_gt_reset(engine->gt, engine->mask);  in enable_error_interrupt()
4100 	 * - I915_ERROR_INSTUCTION (bit 0)  in enable_error_interrupt()
4107 	 * - CP_PRIV (bit 2)  in enable_error_interrupt()
4110 	 *    the instruction with a no-op). This also fires for writes into  in enable_error_interrupt()
4111 	 *    read-only scratch pages.  in enable_error_interrupt()
4113 	 *    This is a non-fatal error, parsing continues.  in enable_error_interrupt()
4128 	assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);  in enable_execlists()
4132 	if (INTEL_GEN(engine->i915) >= 11)  in enable_execlists()
4142 			i915_ggtt_offset(engine->status_page.vma));  in enable_execlists()
4147 	engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0);  in enable_execlists()
4155 		drm_dbg(&engine->i915->drm,  in unexpected_starting_state()
4167 	intel_breadcrumbs_reset(engine->breadcrumbs);  in execlists_resume()
4182 	struct intel_engine_execlists * const execlists = &engine->execlists;  in execlists_reset_prepare()
4185 	ENGINE_TRACE(engine, "depth<-%d\n",  in execlists_reset_prepare()
4186 		     atomic_read(&execlists->tasklet.count));  in execlists_reset_prepare()
4192 	 * to a second via its execlists->tasklet *just* as we are  in execlists_reset_prepare()
4193 	 * calling engine->resume() and also writing the ELSP.  in execlists_reset_prepare()
4194 	 * Turning off the execlists->tasklet until the reset is over  in execlists_reset_prepare()
4197 	__tasklet_disable_sync_once(&execlists->tasklet);  in execlists_reset_prepare()
4201 	spin_lock_irqsave(&engine->active.lock, flags);  in execlists_reset_prepare()
4202 	spin_unlock_irqrestore(&engine->active.lock, flags);  in execlists_reset_prepare()
4219 	engine->execlists.reset_ccid = active_ccid(engine);  in execlists_reset_prepare()
4224 	int x;  in __reset_stop_ring()  local
4226 	x = lrc_ring_mi_mode(engine);  in __reset_stop_ring()
4227 	if (x != -1) {  in __reset_stop_ring()
4228 		regs[x + 1] &= ~STOP_RING;  in __reset_stop_ring()
4229 		regs[x + 1] |= STOP_RING << 16;  in __reset_stop_ring()
4236 	u32 *regs = ce->lrc_reg_state;  in __execlists_reset_reg_state()
4243 	struct intel_engine_execlists * const execlists = &engine->execlists;  in __execlists_reset()
4249 	clflush(execlists->csb_write);  in __execlists_reset()
4262 	rq = active_context(engine, engine->execlists.reset_ccid);  in __execlists_reset()
4266 	ce = rq->context;  in __execlists_reset()
4267 	GEM_BUG_ON(!i915_vma_is_pinned(ce->state));  in __execlists_reset()
4271 		head = intel_ring_wrap(ce->ring, rq->tail);  in __execlists_reset()
4275 	/* We still have requests in-flight; the engine should be active */  in __execlists_reset()
4278 	/* Context has requests still in-flight; it should not be idle! */  in __execlists_reset()
4279 	GEM_BUG_ON(i915_active_is_idle(&ce->active));  in __execlists_reset()
4281 	rq = active_request(ce->timeline, rq);  in __execlists_reset()
4282 	head = intel_ring_wrap(ce->ring, rq->head);  in __execlists_reset()
4283 	GEM_BUG_ON(head == ce->ring->tail);  in __execlists_reset()
4322 	ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n",  in __execlists_reset()
4323 		     head, ce->ring->tail);  in __execlists_reset()
4326 	ce->lrc.desc |= CTX_DESC_FORCE_RESTORE; /* paranoid: GPU was reset! */  in __execlists_reset()
4340 	spin_lock_irqsave(&engine->active.lock, flags);  in execlists_reset_rewind()
4344 	spin_unlock_irqrestore(&engine->active.lock, flags);  in execlists_reset_rewind()
4352 	WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN);  in nop_submission_tasklet()
4357 	struct intel_engine_execlists * const execlists = &engine->execlists;  in execlists_reset_cancel()
4365 	 * Before we call engine->cancel_requests(), we should have exclusive  in execlists_reset_cancel()
4378 	spin_lock_irqsave(&engine->active.lock, flags);  in execlists_reset_cancel()
4383 	list_for_each_entry(rq, &engine->active.requests, sched.link)  in execlists_reset_cancel()
4387 	while ((rb = rb_first_cached(&execlists->queue))) {  in execlists_reset_cancel()
4396 		rb_erase_cached(&p->node, &execlists->queue);  in execlists_reset_cancel()
4400 	/* On-hold requests will be flushed to timeline upon their release */  in execlists_reset_cancel()
4401 	list_for_each_entry(rq, &engine->active.hold, sched.link)  in execlists_reset_cancel()
4405 	while ((rb = rb_first_cached(&execlists->virtual))) {  in execlists_reset_cancel()
4407 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in execlists_reset_cancel()
4409 		rb_erase_cached(rb, &execlists->virtual);  in execlists_reset_cancel()
4412 		spin_lock(&ve->base.active.lock);  in execlists_reset_cancel()
4413 		rq = fetch_and_zero(&ve->request);  in execlists_reset_cancel()
4417 			rq->engine = engine;  in execlists_reset_cancel()
4421 			ve->base.execlists.queue_priority_hint = INT_MIN;  in execlists_reset_cancel()
4423 		spin_unlock(&ve->base.active.lock);  in execlists_reset_cancel()
4428 	execlists->queue_priority_hint = INT_MIN;  in execlists_reset_cancel()
4429 	execlists->queue = RB_ROOT_CACHED;  in execlists_reset_cancel()
4431 	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));  in execlists_reset_cancel()
4432 	execlists->tasklet.func = nop_submission_tasklet;  in execlists_reset_cancel()
4434 	spin_unlock_irqrestore(&engine->active.lock, flags);  in execlists_reset_cancel()
4439 	struct intel_engine_execlists * const execlists = &engine->execlists;  in execlists_reset_finish()
4447 	if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))  in execlists_reset_finish()
4448 		execlists->tasklet.func(execlists->tasklet.data);  in execlists_reset_finish()
4450 	if (__tasklet_enable(&execlists->tasklet))  in execlists_reset_finish()
4452 		tasklet_hi_schedule(&execlists->tasklet);  in execlists_reset_finish()
4453 	ENGINE_TRACE(engine, "depth->%d\n",  in execlists_reset_finish()
4454 		     atomic_read(&execlists->tasklet.count));  in execlists_reset_finish()
4461 	u32 *cs;  in gen8_emit_bb_start_noarb()  local
4463 	cs = intel_ring_begin(rq, 4);  in gen8_emit_bb_start_noarb()
4464 	if (IS_ERR(cs))  in gen8_emit_bb_start_noarb()
4465 		return PTR_ERR(cs);  in gen8_emit_bb_start_noarb()
4477 	 * re-enabled before we close the request  in gen8_emit_bb_start_noarb()
4478 	 * (engine->emit_fini_breadcrumb).  in gen8_emit_bb_start_noarb()
4480 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start_noarb()
4483 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start_noarb()
4485 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start_noarb()
4486 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start_noarb()
4488 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start_noarb()
4497 	u32 *cs;  in gen8_emit_bb_start()  local
4499 	cs = intel_ring_begin(rq, 6);  in gen8_emit_bb_start()
4500 	if (IS_ERR(cs))  in gen8_emit_bb_start()
4501 		return PTR_ERR(cs);  in gen8_emit_bb_start()
4503 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_bb_start()
4505 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start()
4507 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start()
4508 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start()
4510 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start()
4511 	*cs++ = MI_NOOP;  in gen8_emit_bb_start()
4513 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start()
4521 		     ~(engine->irq_enable_mask | engine->irq_keep_mask));  in gen8_logical_ring_enable_irq()
4527 	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);  in gen8_logical_ring_disable_irq()
4532 	u32 cmd, *cs;  in gen8_emit_flush()  local
4534 	cs = intel_ring_begin(request, 4);  in gen8_emit_flush()
4535 	if (IS_ERR(cs))  in gen8_emit_flush()
4536 		return PTR_ERR(cs);  in gen8_emit_flush()
4549 		if (request->engine->class == VIDEO_DECODE_CLASS)  in gen8_emit_flush()
4553 	*cs++ = cmd;  in gen8_emit_flush()
4554 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen8_emit_flush()
4555 	*cs++ = 0; /* upper addr */  in gen8_emit_flush()
4556 	*cs++ = 0; /* value */  in gen8_emit_flush()
4557 	intel_ring_advance(request, cs);  in gen8_emit_flush()
4566 	u32 *cs, flags = 0;  in gen8_emit_flush_render()  local
4592 		if (IS_GEN(request->engine->i915, 9))  in gen8_emit_flush_render()
4596 		if (IS_KBL_GT_REVID(request->engine->i915, 0, KBL_REVID_B0))  in gen8_emit_flush_render()
4608 	cs = intel_ring_begin(request, len);  in gen8_emit_flush_render()
4609 	if (IS_ERR(cs))  in gen8_emit_flush_render()
4610 		return PTR_ERR(cs);  in gen8_emit_flush_render()
4613 		cs = gen8_emit_pipe_control(cs, 0, 0);  in gen8_emit_flush_render()
4616 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,  in gen8_emit_flush_render()
4619 	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen8_emit_flush_render()
4622 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);  in gen8_emit_flush_render()
4624 	intel_ring_advance(request, cs);  in gen8_emit_flush_render()
4633 		u32 *cs;  in gen11_emit_flush_render()  local
4646 		cs = intel_ring_begin(request, 6);  in gen11_emit_flush_render()
4647 		if (IS_ERR(cs))  in gen11_emit_flush_render()
4648 			return PTR_ERR(cs);  in gen11_emit_flush_render()
4650 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_render()
4651 		intel_ring_advance(request, cs);  in gen11_emit_flush_render()
4655 		u32 *cs;  in gen11_emit_flush_render()  local
4670 		cs = intel_ring_begin(request, 6);  in gen11_emit_flush_render()
4671 		if (IS_ERR(cs))  in gen11_emit_flush_render()
4672 			return PTR_ERR(cs);  in gen11_emit_flush_render()
4674 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_render()
4675 		intel_ring_advance(request, cs);  in gen11_emit_flush_render()
4700 	if (engine->class == VIDEO_DECODE_CLASS)  in aux_inv_reg()
4701 		return vd[engine->instance];  in aux_inv_reg()
4703 	if (engine->class == VIDEO_ENHANCEMENT_CLASS)  in aux_inv_reg()
4704 		return ve[engine->instance];  in aux_inv_reg()
4712 gen12_emit_aux_table_inv(const i915_reg_t inv_reg, u32 *cs)  in gen12_emit_aux_table_inv()  argument
4714 	*cs++ = MI_LOAD_REGISTER_IMM(1);  in gen12_emit_aux_table_inv()
4715 	*cs++ = i915_mmio_reg_offset(inv_reg);  in gen12_emit_aux_table_inv()
4716 	*cs++ = AUX_INV;  in gen12_emit_aux_table_inv()
4717 	*cs++ = MI_NOOP;  in gen12_emit_aux_table_inv()
4719 	return cs;  in gen12_emit_aux_table_inv()
4727 		u32 *cs;  in gen12_emit_flush_render()  local
4743 		cs = intel_ring_begin(request, 6);  in gen12_emit_flush_render()
4744 		if (IS_ERR(cs))  in gen12_emit_flush_render()
4745 			return PTR_ERR(cs);  in gen12_emit_flush_render()
4747 		cs = gen12_emit_pipe_control(cs,  in gen12_emit_flush_render()
4750 		intel_ring_advance(request, cs);  in gen12_emit_flush_render()
4755 		u32 *cs;  in gen12_emit_flush_render()  local
4770 		cs = intel_ring_begin(request, 8 + 4);  in gen12_emit_flush_render()
4771 		if (IS_ERR(cs))  in gen12_emit_flush_render()
4772 			return PTR_ERR(cs);  in gen12_emit_flush_render()
4775 		 * Prevent the pre-parser from skipping past the TLB  in gen12_emit_flush_render()
4779 		*cs++ = preparser_disable(true);  in gen12_emit_flush_render()
4781 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen12_emit_flush_render()
4784 		cs = gen12_emit_aux_table_inv(GEN12_GFX_CCS_AUX_NV, cs);  in gen12_emit_flush_render()
4786 		*cs++ = preparser_disable(false);  in gen12_emit_flush_render()
4787 		intel_ring_advance(request, cs);  in gen12_emit_flush_render()
4796 	u32 cmd, *cs;  in gen12_emit_flush()  local
4802 		aux_inv = request->engine->mask & ~BIT(BCS0);  in gen12_emit_flush()
4806 	cs = intel_ring_begin(request, cmd);  in gen12_emit_flush()
4807 	if (IS_ERR(cs))  in gen12_emit_flush()
4808 		return PTR_ERR(cs);  in gen12_emit_flush()
4811 		*cs++ = preparser_disable(true);  in gen12_emit_flush()
4824 		if (request->engine->class == VIDEO_DECODE_CLASS)  in gen12_emit_flush()
4828 	*cs++ = cmd;  in gen12_emit_flush()
4829 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen12_emit_flush()
4830 	*cs++ = 0; /* upper addr */  in gen12_emit_flush()
4831 	*cs++ = 0; /* value */  in gen12_emit_flush()
4837 		*cs++ = MI_LOAD_REGISTER_IMM(hweight8(aux_inv));  in gen12_emit_flush()
4838 		for_each_engine_masked(engine, request->engine->gt,  in gen12_emit_flush()
4840 			*cs++ = i915_mmio_reg_offset(aux_inv_reg(engine));  in gen12_emit_flush()
4841 			*cs++ = AUX_INV;  in gen12_emit_flush()
4843 		*cs++ = MI_NOOP;  in gen12_emit_flush()
4847 		*cs++ = preparser_disable(false);  in gen12_emit_flush()
4849 	intel_ring_advance(request, cs);  in gen12_emit_flush()
4856 	struct intel_ring *ring __maybe_unused = rq->ring;  in assert_request_valid()
4859 	GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0);  in assert_request_valid()
4867 static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs)  in gen8_emit_wa_tail()  argument
4869 	/* Ensure there's always at least one preemption point per-request. */  in gen8_emit_wa_tail()
4870 	*cs++ = MI_ARB_CHECK;  in gen8_emit_wa_tail()
4871 	*cs++ = MI_NOOP;  in gen8_emit_wa_tail()
4872 	request->wa_tail = intel_ring_offset(request, cs);  in gen8_emit_wa_tail()
4877 	return cs;  in gen8_emit_wa_tail()
4880 static u32 *emit_preempt_busywait(struct i915_request *request, u32 *cs)  in emit_preempt_busywait()  argument
4882 	*cs++ = MI_SEMAPHORE_WAIT |  in emit_preempt_busywait()
4886 	*cs++ = 0;  in emit_preempt_busywait()
4887 	*cs++ = intel_hws_preempt_address(request->engine);  in emit_preempt_busywait()
4888 	*cs++ = 0;  in emit_preempt_busywait()
4890 	return cs;  in emit_preempt_busywait()
4894 gen8_emit_fini_breadcrumb_tail(struct i915_request *request, u32 *cs)  in gen8_emit_fini_breadcrumb_tail()  argument
4896 	*cs++ = MI_USER_INTERRUPT;  in gen8_emit_fini_breadcrumb_tail()
4898 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_fini_breadcrumb_tail()
4899 	if (intel_engine_has_semaphores(request->engine))  in gen8_emit_fini_breadcrumb_tail()
4900 		cs = emit_preempt_busywait(request, cs);  in gen8_emit_fini_breadcrumb_tail()
4902 	request->tail = intel_ring_offset(request, cs);  in gen8_emit_fini_breadcrumb_tail()
4903 	assert_ring_tail_valid(request->ring, request->tail);  in gen8_emit_fini_breadcrumb_tail()
4905 	return gen8_emit_wa_tail(request, cs);  in gen8_emit_fini_breadcrumb_tail()
4908 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)  in emit_xcs_breadcrumb()  argument
4910 	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);  in emit_xcs_breadcrumb()
4913 static u32 *gen8_emit_fini_breadcrumb(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb()  argument
4915 	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));  in gen8_emit_fini_breadcrumb()
4918 static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs)  in gen8_emit_fini_breadcrumb_rcs()  argument
4920 	cs = gen8_emit_pipe_control(cs,  in gen8_emit_fini_breadcrumb_rcs()
4927 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen8_emit_fini_breadcrumb_rcs()
4928 				      request->fence.seqno,  in gen8_emit_fini_breadcrumb_rcs()
4933 	return gen8_emit_fini_breadcrumb_tail(request, cs);  in gen8_emit_fini_breadcrumb_rcs()
4937 gen11_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs)  in gen11_emit_fini_breadcrumb_rcs()  argument
4939 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen11_emit_fini_breadcrumb_rcs()
4940 				      request->fence.seqno,  in gen11_emit_fini_breadcrumb_rcs()
4949 	return gen8_emit_fini_breadcrumb_tail(request, cs);  in gen11_emit_fini_breadcrumb_rcs()
4953  * Note that the CS instruction pre-parser will not stall on the breadcrumb
4954  * flush and will continue pre-fetching the instructions after it before the
4955  * memory sync is completed. On pre-gen12 HW, the pre-parser will stop at
4956  * BB_START/END instructions, so, even though we might pre-fetch the pre-amble
4959  * However, on gen12+ the parser can pre-fetch across the BB_START/END commands,
4961  * the same intel_context, we might pre-fetch and then execute the pre-update
4962  * instruction. To avoid this, the users of self-modifying code should either
4965  * the in-kernel use-cases we've opted to use a separate context, see
4967  * All the above applies only to the instructions themselves. Non-inline data
4968  * used by the instructions is not pre-fetched.
4971 static u32 *gen12_emit_preempt_busywait(struct i915_request *request, u32 *cs)  in gen12_emit_preempt_busywait()  argument
4973 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_preempt_busywait()
4977 	*cs++ = 0;  in gen12_emit_preempt_busywait()
4978 	*cs++ = intel_hws_preempt_address(request->engine);  in gen12_emit_preempt_busywait()
4979 	*cs++ = 0;  in gen12_emit_preempt_busywait()
4980 	*cs++ = 0;  in gen12_emit_preempt_busywait()
4981 	*cs++ = MI_NOOP;  in gen12_emit_preempt_busywait()
4983 	return cs;  in gen12_emit_preempt_busywait()
4987 gen12_emit_fini_breadcrumb_tail(struct i915_request *request, u32 *cs)  in gen12_emit_fini_breadcrumb_tail()  argument
4989 	*cs++ = MI_USER_INTERRUPT;  in gen12_emit_fini_breadcrumb_tail()
4991 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen12_emit_fini_breadcrumb_tail()
4992 	if (intel_engine_has_semaphores(request->engine))  in gen12_emit_fini_breadcrumb_tail()
4993 		cs = gen12_emit_preempt_busywait(request, cs);  in gen12_emit_fini_breadcrumb_tail()
4995 	request->tail = intel_ring_offset(request, cs);  in gen12_emit_fini_breadcrumb_tail()
4996 	assert_ring_tail_valid(request->ring, request->tail);  in gen12_emit_fini_breadcrumb_tail()
4998 	return gen8_emit_wa_tail(request, cs);  in gen12_emit_fini_breadcrumb_tail()
5001 static u32 *gen12_emit_fini_breadcrumb(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb()  argument
5003 	/* XXX Stalling flush before seqno write; post-sync not */  in gen12_emit_fini_breadcrumb()
5004 	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));  in gen12_emit_fini_breadcrumb()
5005 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb()
5009 gen12_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs)  in gen12_emit_fini_breadcrumb_rcs()  argument
5011 	cs = gen12_emit_ggtt_write_rcs(cs,  in gen12_emit_fini_breadcrumb_rcs()
5012 				       request->fence.seqno,  in gen12_emit_fini_breadcrumb_rcs()
5025 	return gen12_emit_fini_breadcrumb_tail(request, cs);  in gen12_emit_fini_breadcrumb_rcs()
5030 	cancel_timer(&engine->execlists.timer);  in execlists_park()
5031 	cancel_timer(&engine->execlists.preempt);  in execlists_park()
5036 	engine->submit_request = execlists_submit_request;  in intel_execlists_set_default_submission()
5037 	engine->schedule = i915_schedule;  in intel_execlists_set_default_submission()
5038 	engine->execlists.tasklet.func = execlists_submission_tasklet;  in intel_execlists_set_default_submission()
5040 	engine->reset.prepare = execlists_reset_prepare;  in intel_execlists_set_default_submission()
5041 	engine->reset.rewind = execlists_reset_rewind;  in intel_execlists_set_default_submission()
5042 	engine->reset.cancel = execlists_reset_cancel;  in intel_execlists_set_default_submission()
5043 	engine->reset.finish = execlists_reset_finish;  in intel_execlists_set_default_submission()
5045 	engine->park = execlists_park;  in intel_execlists_set_default_submission()
5046 	engine->unpark = NULL;  in intel_execlists_set_default_submission()
5048 	engine->flags |= I915_ENGINE_SUPPORTS_STATS;  in intel_execlists_set_default_submission()
5049 	if (!intel_vgpu_active(engine->i915)) {  in intel_execlists_set_default_submission()
5050 		engine->flags |= I915_ENGINE_HAS_SEMAPHORES;  in intel_execlists_set_default_submission()
5051 		if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) {  in intel_execlists_set_default_submission()
5052 			engine->flags |= I915_ENGINE_HAS_PREEMPTION;  in intel_execlists_set_default_submission()
5054 				engine->flags |= I915_ENGINE_HAS_TIMESLICES;  in intel_execlists_set_default_submission()
5058 	if (INTEL_GEN(engine->i915) >= 12)  in intel_execlists_set_default_submission()
5059 		engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO;  in intel_execlists_set_default_submission()
5062 		engine->emit_bb_start = gen8_emit_bb_start;  in intel_execlists_set_default_submission()
5064 		engine->emit_bb_start = gen8_emit_bb_start_noarb;  in intel_execlists_set_default_submission()
5070 	del_timer_sync(&engine->execlists.timer);  in execlists_shutdown()
5071 	del_timer_sync(&engine->execlists.preempt);  in execlists_shutdown()
5072 	tasklet_kill(&engine->execlists.tasklet);  in execlists_shutdown()
5077 	engine->sanitize = NULL; /* no longer in control, nothing to sanitize */  in execlists_release()
5090 	engine->resume = execlists_resume;  in logical_ring_default_vfuncs()
5092 	engine->cops = &execlists_context_ops;  in logical_ring_default_vfuncs()
5093 	engine->request_alloc = execlists_request_alloc;  in logical_ring_default_vfuncs()
5095 	engine->emit_flush = gen8_emit_flush;  in logical_ring_default_vfuncs()
5096 	engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;  in logical_ring_default_vfuncs()
5097 	engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb;  in logical_ring_default_vfuncs()
5098 	if (INTEL_GEN(engine->i915) >= 12) {  in logical_ring_default_vfuncs()
5099 		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb;  in logical_ring_default_vfuncs()
5100 		engine->emit_flush = gen12_emit_flush;  in logical_ring_default_vfuncs()
5102 	engine->set_default_submission = intel_execlists_set_default_submission;  in logical_ring_default_vfuncs()
5104 	if (INTEL_GEN(engine->i915) < 11) {  in logical_ring_default_vfuncs()
5105 		engine->irq_enable = gen8_logical_ring_enable_irq;  in logical_ring_default_vfuncs()
5106 		engine->irq_disable = gen8_logical_ring_disable_irq;  in logical_ring_default_vfuncs()
5122 	if (INTEL_GEN(engine->i915) < 11) {  in logical_ring_default_irqs()
5131 		shift = irq_shifts[engine->id];  in logical_ring_default_irqs()
5134 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;  in logical_ring_default_irqs()
5135 	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;  in logical_ring_default_irqs()
5136 	engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift;  in logical_ring_default_irqs()
5137 	engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift;  in logical_ring_default_irqs()
5142 	switch (INTEL_GEN(engine->i915)) {  in rcs_submission_override()
5144 		engine->emit_flush = gen12_emit_flush_render;  in rcs_submission_override()
5145 		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;  in rcs_submission_override()
5148 		engine->emit_flush = gen11_emit_flush_render;  in rcs_submission_override()
5149 		engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;  in rcs_submission_override()
5152 		engine->emit_flush = gen8_emit_flush_render;  in rcs_submission_override()
5153 		engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;  in rcs_submission_override()
5160 	struct intel_engine_execlists * const execlists = &engine->execlists;  in intel_execlists_submission_setup()
5161 	struct drm_i915_private *i915 = engine->i915;  in intel_execlists_submission_setup()
5162 	struct intel_uncore *uncore = engine->uncore;  in intel_execlists_submission_setup()
5163 	u32 base = engine->mmio_base;  in intel_execlists_submission_setup()
5165 	tasklet_init(&engine->execlists.tasklet,  in intel_execlists_submission_setup()
5167 	timer_setup(&engine->execlists.timer, execlists_timeslice, 0);  in intel_execlists_submission_setup()
5168 	timer_setup(&engine->execlists.preempt, execlists_preempt, 0);  in intel_execlists_submission_setup()
5173 	if (engine->class == RENDER_CLASS)  in intel_execlists_submission_setup()
5182 		drm_err(&i915->drm, "WA batch buffer initialization failed\n");  in intel_execlists_submission_setup()
5185 		execlists->submit_reg = uncore->regs +  in intel_execlists_submission_setup()
5187 		execlists->ctrl_reg = uncore->regs +  in intel_execlists_submission_setup()
5190 		execlists->submit_reg = uncore->regs +  in intel_execlists_submission_setup()
5194 	execlists->csb_status =  in intel_execlists_submission_setup()
5195 		(u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];  in intel_execlists_submission_setup()
5197 	execlists->csb_write =  in intel_execlists_submission_setup()
5198 		&engine->status_page.addr[intel_hws_csb_write_index(i915)];  in intel_execlists_submission_setup()
5201 		execlists->csb_size = GEN8_CSB_ENTRIES;  in intel_execlists_submission_setup()
5203 		execlists->csb_size = GEN11_CSB_ENTRIES;  in intel_execlists_submission_setup()
5205 	if (INTEL_GEN(engine->i915) >= 11) {  in intel_execlists_submission_setup()
5206 		execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32);  in intel_execlists_submission_setup()
5207 		execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32);  in intel_execlists_submission_setup()
5211 	engine->sanitize = execlists_sanitize;  in intel_execlists_submission_setup()
5212 	engine->release = execlists_release;  in intel_execlists_submission_setup()
5228 	if (INTEL_GEN(engine->i915) < 11)  in init_common_reg_state()
5233 	regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID;  in init_common_reg_state()
5240 	const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx;  in init_wa_bb_reg_state()
5242 	if (wa_ctx->per_ctx.size) {  in init_wa_bb_reg_state()
5243 		const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);  in init_wa_bb_reg_state()
5245 		GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1);  in init_wa_bb_reg_state()
5247 			(ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;  in init_wa_bb_reg_state()
5250 	if (wa_ctx->indirect_ctx.size) {  in init_wa_bb_reg_state()
5252 					    i915_ggtt_offset(wa_ctx->vma) +  in init_wa_bb_reg_state()
5253 					    wa_ctx->indirect_ctx.offset,  in init_wa_bb_reg_state()
5254 					    wa_ctx->indirect_ctx.size);  in init_wa_bb_reg_state()
5260 	if (i915_vm_is_4lvl(&ppgtt->vm)) {  in init_ppgtt_reg_state()
5277 		return i915_vm_to_ggtt(vm)->alias;  in vm_alias()
5301 	init_ppgtt_reg_state(regs, vm_alias(ce->vm));  in execlists_init_reg_state()
5319 		drm_dbg(&engine->i915->drm, "Could not map object pages!\n");  in populate_lr_context()
5325 	if (engine->default_state) {  in populate_lr_context()
5326 		shmem_read(engine->default_state, 0,  in populate_lr_context()
5327 			   vaddr, engine->context_size);  in populate_lr_context()
5328 		__set_bit(CONTEXT_VALID_BIT, &ce->flags);  in populate_lr_context()
5332 	/* Clear the ppHWSP (inc. per-context counters) */  in populate_lr_context()
5342 	__i915_gem_object_flush_map(ctx_obj, 0, engine->context_size);  in populate_lr_context()
5349 	struct intel_timeline *tl = fetch_and_zero(&ce->timeline);  in pinned_timeline()
5351 	return intel_timeline_create_from_engine(ce->engine,  in pinned_timeline()
5364 	GEM_BUG_ON(ce->state);  in __execlists_context_alloc()
5365 	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);  in __execlists_context_alloc()
5370 	if (INTEL_GEN(engine->i915) == 12) {  in __execlists_context_alloc()
5371 		ce->wa_bb_page = context_size / PAGE_SIZE;  in __execlists_context_alloc()
5375 	ctx_obj = i915_gem_object_create_shmem(engine->i915, context_size);  in __execlists_context_alloc()
5379 	vma = i915_vma_instance(ctx_obj, &engine->gt->ggtt->vm, NULL);  in __execlists_context_alloc()
5385 	if (!page_mask_bits(ce->timeline)) {  in __execlists_context_alloc()
5392 		if (unlikely(ce->timeline))  in __execlists_context_alloc()
5395 			tl = intel_timeline_create(engine->gt);  in __execlists_context_alloc()
5401 		ce->timeline = tl;  in __execlists_context_alloc()
5404 	ring = intel_engine_create_ring(engine, (unsigned long)ce->ring);  in __execlists_context_alloc()
5412 		drm_dbg(&engine->i915->drm,  in __execlists_context_alloc()
5417 	ce->ring = ring;  in __execlists_context_alloc()
5418 	ce->state = vma;  in __execlists_context_alloc()
5431 	return &ve->base.execlists.default_priolist.requests[0];  in virtual_queue()
5440 	GEM_BUG_ON(ve->context.inflight);  in rcu_virtual_context_destroy()
5442 	/* Preempt-to-busy may leave a stale request behind. */  in rcu_virtual_context_destroy()
5443 	if (unlikely(ve->request)) {  in rcu_virtual_context_destroy()
5446 		spin_lock_irq(&ve->base.active.lock);  in rcu_virtual_context_destroy()
5448 		old = fetch_and_zero(&ve->request);  in rcu_virtual_context_destroy()
5455 		spin_unlock_irq(&ve->base.active.lock);  in rcu_virtual_context_destroy()
5465 	tasklet_kill(&ve->base.execlists.tasklet);  in rcu_virtual_context_destroy()
5468 	for (n = 0; n < ve->num_siblings; n++) {  in rcu_virtual_context_destroy()
5469 		struct intel_engine_cs *sibling = ve->siblings[n];  in rcu_virtual_context_destroy()
5470 		struct rb_node *node = &ve->nodes[sibling->id].rb;  in rcu_virtual_context_destroy()
5475 		spin_lock_irq(&sibling->active.lock);  in rcu_virtual_context_destroy()
5479 			rb_erase_cached(node, &sibling->execlists.virtual);  in rcu_virtual_context_destroy()
5481 		spin_unlock_irq(&sibling->active.lock);  in rcu_virtual_context_destroy()
5483 	GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet));  in rcu_virtual_context_destroy()
5486 	if (ve->context.state)  in rcu_virtual_context_destroy()
5487 		__execlists_context_fini(&ve->context);  in rcu_virtual_context_destroy()
5488 	intel_context_fini(&ve->context);  in rcu_virtual_context_destroy()
5490 	intel_breadcrumbs_free(ve->base.breadcrumbs);  in rcu_virtual_context_destroy()
5491 	intel_engine_free_request_pool(&ve->base);  in rcu_virtual_context_destroy()
5493 	kfree(ve->bonds);  in rcu_virtual_context_destroy()
5502 	GEM_BUG_ON(!list_empty(&ve->context.signals));  in virtual_context_destroy()
5508 	 * due to preempt-to-busy). Before we can free the engine, we need  in virtual_context_destroy()
5514 	INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy);  in virtual_context_destroy()
5515 	queue_rcu_work(system_wq, &ve->rcu);  in virtual_context_destroy()
5535 	swp = prandom_u32_max(ve->num_siblings);  in virtual_engine_initial_hint()
5537 		swap(ve->siblings[swp], ve->siblings[0]);  in virtual_engine_initial_hint()
5544 	return __execlists_context_alloc(ce, ve->siblings[0]);  in virtual_context_alloc()
5552 	return __execlists_context_pin(ce, ve->siblings[0], vaddr);  in virtual_context_pin()
5560 	for (n = 0; n < ve->num_siblings; n++)  in virtual_context_enter()
5561 		intel_engine_pm_get(ve->siblings[n]);  in virtual_context_enter()
5563 	intel_timeline_enter(ce->timeline);  in virtual_context_enter()
5571 	intel_timeline_exit(ce->timeline);  in virtual_context_exit()
5573 	for (n = 0; n < ve->num_siblings; n++)  in virtual_context_exit()
5574 		intel_engine_pm_put(ve->siblings[n]);  in virtual_context_exit()
5596 	rq = READ_ONCE(ve->request);  in virtual_submission_mask()
5600 	/* The rq is ready for submission; rq->execution_mask is now stable. */  in virtual_submission_mask()
5601 	mask = rq->execution_mask;  in virtual_submission_mask()
5604 		i915_request_set_error_once(rq, -ENODEV);  in virtual_submission_mask()
5605 		mask = ve->siblings[0]->mask;  in virtual_submission_mask()
5608 	ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",  in virtual_submission_mask()
5609 		     rq->fence.context, rq->fence.seqno,  in virtual_submission_mask()
5610 		     mask, ve->base.execlists.queue_priority_hint);  in virtual_submission_mask()
5618 	const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint);  in virtual_submission_tasklet()
5629 	for (n = 0; n < ve->num_siblings; n++) {  in virtual_submission_tasklet()
5630 		struct intel_engine_cs *sibling = READ_ONCE(ve->siblings[n]);  in virtual_submission_tasklet()
5631 		struct ve_node * const node = &ve->nodes[sibling->id];  in virtual_submission_tasklet()
5635 		if (!READ_ONCE(ve->request))  in virtual_submission_tasklet()
5638 		if (unlikely(!(mask & sibling->mask))) {  in virtual_submission_tasklet()
5639 			if (!RB_EMPTY_NODE(&node->rb)) {  in virtual_submission_tasklet()
5640 				spin_lock(&sibling->active.lock);  in virtual_submission_tasklet()
5641 				rb_erase_cached(&node->rb,  in virtual_submission_tasklet()
5642 						&sibling->execlists.virtual);  in virtual_submission_tasklet()
5643 				RB_CLEAR_NODE(&node->rb);  in virtual_submission_tasklet()
5644 				spin_unlock(&sibling->active.lock);  in virtual_submission_tasklet()
5649 		spin_lock(&sibling->active.lock);  in virtual_submission_tasklet()
5651 		if (!RB_EMPTY_NODE(&node->rb)) {  in virtual_submission_tasklet()
5656 			first = rb_first_cached(&sibling->execlists.virtual) ==  in virtual_submission_tasklet()
5657 				&node->rb;  in virtual_submission_tasklet()
5658 			if (prio == node->prio || (prio > node->prio && first))  in virtual_submission_tasklet()
5661 			rb_erase_cached(&node->rb, &sibling->execlists.virtual);  in virtual_submission_tasklet()
5666 		parent = &sibling->execlists.virtual.rb_root.rb_node;  in virtual_submission_tasklet()
5672 			if (prio > other->prio) {  in virtual_submission_tasklet()
5673 				parent = &rb->rb_left;  in virtual_submission_tasklet()
5675 				parent = &rb->rb_right;  in virtual_submission_tasklet()
5680 		rb_link_node(&node->rb, rb, parent);  in virtual_submission_tasklet()
5681 		rb_insert_color_cached(&node->rb,  in virtual_submission_tasklet()
5682 				       &sibling->execlists.virtual,  in virtual_submission_tasklet()
5686 		GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));  in virtual_submission_tasklet()
5687 		node->prio = prio;  in virtual_submission_tasklet()
5688 		if (first && prio > sibling->execlists.queue_priority_hint)  in virtual_submission_tasklet()
5689 			tasklet_hi_schedule(&sibling->execlists.tasklet);  in virtual_submission_tasklet()
5691 		spin_unlock(&sibling->active.lock);  in virtual_submission_tasklet()
5698 	struct virtual_engine *ve = to_virtual_engine(rq->engine);  in virtual_submit_request()
5702 	ENGINE_TRACE(&ve->base, "rq=%llx:%lld\n",  in virtual_submit_request()
5703 		     rq->fence.context,  in virtual_submit_request()
5704 		     rq->fence.seqno);  in virtual_submit_request()
5706 	GEM_BUG_ON(ve->base.submit_request != virtual_submit_request);  in virtual_submit_request()
5708 	spin_lock_irqsave(&ve->base.active.lock, flags);  in virtual_submit_request()
5710 	old = ve->request;  in virtual_submit_request()
5711 	if (old) { /* background completion event from preempt-to-busy */  in virtual_submit_request()
5720 		ve->base.execlists.queue_priority_hint = INT_MIN;  in virtual_submit_request()
5721 		ve->request = NULL;  in virtual_submit_request()
5723 		ve->base.execlists.queue_priority_hint = rq_prio(rq);  in virtual_submit_request()
5724 		ve->request = i915_request_get(rq);  in virtual_submit_request()
5727 		list_move_tail(&rq->sched.link, virtual_queue(ve));  in virtual_submit_request()
5729 		tasklet_hi_schedule(&ve->base.execlists.tasklet);  in virtual_submit_request()
5732 	spin_unlock_irqrestore(&ve->base.active.lock, flags);  in virtual_submit_request()
5741 	for (i = 0; i < ve->num_bonds; i++) {  in virtual_find_bond()
5742 		if (ve->bonds[i].master == master)  in virtual_find_bond()
5743 			return &ve->bonds[i];  in virtual_find_bond()
5752 	struct virtual_engine *ve = to_virtual_engine(rq->engine);  in virtual_bond_execute()
5756 	allowed = ~to_request(signal)->engine->mask;  in virtual_bond_execute()
5758 	bond = virtual_find_bond(ve, to_request(signal)->engine);  in virtual_bond_execute()
5760 		allowed &= bond->sibling_mask;  in virtual_bond_execute()
5763 	exec = READ_ONCE(rq->execution_mask);  in virtual_bond_execute()
5764 	while (!try_cmpxchg(&rq->execution_mask, &exec, exec & allowed))  in virtual_bond_execute()
5767 	/* Prevent the master from being re-run on the bonded engines */  in virtual_bond_execute()
5768 	to_request(signal)->execution_mask &= ~allowed;  in virtual_bond_execute()
5780 		return ERR_PTR(-EINVAL);  in intel_execlists_create_virtual()
5787 		return ERR_PTR(-ENOMEM);  in intel_execlists_create_virtual()
5789 	ve->base.i915 = siblings[0]->i915;  in intel_execlists_create_virtual()
5790 	ve->base.gt = siblings[0]->gt;  in intel_execlists_create_virtual()
5791 	ve->base.uncore = siblings[0]->uncore;  in intel_execlists_create_virtual()
5792 	ve->base.id = -1;  in intel_execlists_create_virtual()
5794 	ve->base.class = OTHER_CLASS;  in intel_execlists_create_virtual()
5795 	ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;  in intel_execlists_create_virtual()
5796 	ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;  in intel_execlists_create_virtual()
5797 	ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;  in intel_execlists_create_virtual()
5807 	 * and be pessimized in priority for doing so -- if we are the only  in intel_execlists_create_virtual()
5812 	ve->base.saturated = ALL_ENGINES;  in intel_execlists_create_virtual()
5814 	snprintf(ve->base.name, sizeof(ve->base.name), "virtual");  in intel_execlists_create_virtual()
5816 	intel_engine_init_active(&ve->base, ENGINE_VIRTUAL);  in intel_execlists_create_virtual()
5817 	intel_engine_init_execlists(&ve->base);  in intel_execlists_create_virtual()
5819 	ve->base.cops = &virtual_context_ops;  in intel_execlists_create_virtual()
5820 	ve->base.request_alloc = execlists_request_alloc;  in intel_execlists_create_virtual()
5822 	ve->base.schedule = i915_schedule;  in intel_execlists_create_virtual()
5823 	ve->base.submit_request = virtual_submit_request;  in intel_execlists_create_virtual()
5824 	ve->base.bond_execute = virtual_bond_execute;  in intel_execlists_create_virtual()
5827 	ve->base.execlists.queue_priority_hint = INT_MIN;  in intel_execlists_create_virtual()
5828 	tasklet_init(&ve->base.execlists.tasklet,  in intel_execlists_create_virtual()
5832 	intel_context_init(&ve->context, &ve->base);  in intel_execlists_create_virtual()
5834 	ve->base.breadcrumbs = intel_breadcrumbs_create(NULL);  in intel_execlists_create_virtual()
5835 	if (!ve->base.breadcrumbs) {  in intel_execlists_create_virtual()
5836 		err = -ENOMEM;  in intel_execlists_create_virtual()
5843 		GEM_BUG_ON(!is_power_of_2(sibling->mask));  in intel_execlists_create_virtual()
5844 		if (sibling->mask & ve->base.mask) {  in intel_execlists_create_virtual()
5846 				  sibling->name);  in intel_execlists_create_virtual()
5847 			err = -EINVAL;  in intel_execlists_create_virtual()
5853 		 * the execlists backend -- we push out request directly  in intel_execlists_create_virtual()
5858 		if (sibling->execlists.tasklet.func !=  in intel_execlists_create_virtual()
5860 			err = -ENODEV;  in intel_execlists_create_virtual()
5864 		GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb));  in intel_execlists_create_virtual()
5865 		RB_CLEAR_NODE(&ve->nodes[sibling->id].rb);  in intel_execlists_create_virtual()
5867 		ve->siblings[ve->num_siblings++] = sibling;  in intel_execlists_create_virtual()
5868 		ve->base.mask |= sibling->mask;  in intel_execlists_create_virtual()
5877 		if (ve->base.class != OTHER_CLASS) {  in intel_execlists_create_virtual()
5878 			if (ve->base.class != sibling->class) {  in intel_execlists_create_virtual()
5880 					  sibling->class, ve->base.class);  in intel_execlists_create_virtual()
5881 				err = -EINVAL;  in intel_execlists_create_virtual()
5887 		ve->base.class = sibling->class;  in intel_execlists_create_virtual()
5888 		ve->base.uabi_class = sibling->uabi_class;  in intel_execlists_create_virtual()
5889 		snprintf(ve->base.name, sizeof(ve->base.name),  in intel_execlists_create_virtual()
5890 			 "v%dx%d", ve->base.class, count);  in intel_execlists_create_virtual()
5891 		ve->base.context_size = sibling->context_size;  in intel_execlists_create_virtual()
5893 		ve->base.emit_bb_start = sibling->emit_bb_start;  in intel_execlists_create_virtual()
5894 		ve->base.emit_flush = sibling->emit_flush;  in intel_execlists_create_virtual()
5895 		ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb;  in intel_execlists_create_virtual()
5896 		ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb;  in intel_execlists_create_virtual()
5897 		ve->base.emit_fini_breadcrumb_dw =  in intel_execlists_create_virtual()
5898 			sibling->emit_fini_breadcrumb_dw;  in intel_execlists_create_virtual()
5900 		ve->base.flags = sibling->flags;  in intel_execlists_create_virtual()
5903 	ve->base.flags |= I915_ENGINE_IS_VIRTUAL;  in intel_execlists_create_virtual()
5906 	return &ve->context;  in intel_execlists_create_virtual()
5909 	intel_context_put(&ve->context);  in intel_execlists_create_virtual()
5919 	dst = intel_execlists_create_virtual(se->siblings,  in intel_execlists_clone_virtual()
5920 					     se->num_siblings);  in intel_execlists_clone_virtual()
5924 	if (se->num_bonds) {  in intel_execlists_clone_virtual()
5925 		struct virtual_engine *de = to_virtual_engine(dst->engine);  in intel_execlists_clone_virtual()
5927 		de->bonds = kmemdup(se->bonds,  in intel_execlists_clone_virtual()
5928 				    sizeof(*se->bonds) * se->num_bonds,  in intel_execlists_clone_virtual()
5930 		if (!de->bonds) {  in intel_execlists_clone_virtual()
5932 			return ERR_PTR(-ENOMEM);  in intel_execlists_clone_virtual()
5935 		de->num_bonds = se->num_bonds;  in intel_execlists_clone_virtual()
5950 	for (n = 0; n < ve->num_siblings; n++)  in intel_virtual_engine_attach_bond()
5951 		if (sibling == ve->siblings[n])  in intel_virtual_engine_attach_bond()
5953 	if (n == ve->num_siblings)  in intel_virtual_engine_attach_bond()
5954 		return -EINVAL;  in intel_virtual_engine_attach_bond()
5958 		bond->sibling_mask |= sibling->mask;  in intel_virtual_engine_attach_bond()
5962 	bond = krealloc(ve->bonds,  in intel_virtual_engine_attach_bond()
5963 			sizeof(*bond) * (ve->num_bonds + 1),  in intel_virtual_engine_attach_bond()
5966 		return -ENOMEM;  in intel_virtual_engine_attach_bond()
5968 	bond[ve->num_bonds].master = master;  in intel_virtual_engine_attach_bond()
5969 	bond[ve->num_bonds].sibling_mask = sibling->mask;  in intel_virtual_engine_attach_bond()
5971 	ve->bonds = bond;  in intel_virtual_engine_attach_bond()
5972 	ve->num_bonds++;  in intel_virtual_engine_attach_bond()
5983 	if (sibling >= ve->num_siblings)  in intel_virtual_engine_get_sibling()
5986 	return ve->siblings[sibling];  in intel_virtual_engine_get_sibling()
5996 	const struct intel_engine_execlists *execlists = &engine->execlists;  in intel_execlists_show_requests()
6002 	spin_lock_irqsave(&engine->active.lock, flags);  in intel_execlists_show_requests()
6006 	list_for_each_entry(rq, &engine->active.requests, sched.link) {  in intel_execlists_show_requests()
6007 		if (count++ < max - 1)  in intel_execlists_show_requests()
6016 				   count - max);  in intel_execlists_show_requests()
6021 	if (execlists->switch_priority_hint != INT_MIN)  in intel_execlists_show_requests()
6023 			   READ_ONCE(execlists->switch_priority_hint));  in intel_execlists_show_requests()
6024 	if (execlists->queue_priority_hint != INT_MIN)  in intel_execlists_show_requests()
6026 			   READ_ONCE(execlists->queue_priority_hint));  in intel_execlists_show_requests()
6030 	for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {  in intel_execlists_show_requests()
6035 			if (count++ < max - 1)  in intel_execlists_show_requests()
6045 				   count - max);  in intel_execlists_show_requests()
6052 	for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) {  in intel_execlists_show_requests()
6054 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);  in intel_execlists_show_requests()
6055 		struct i915_request *rq = READ_ONCE(ve->request);  in intel_execlists_show_requests()
6058 			if (count++ < max - 1)  in intel_execlists_show_requests()
6068 				   count - max);  in intel_execlists_show_requests()
6073 	spin_unlock_irqrestore(&engine->active.lock, flags);  in intel_execlists_show_requests()
6101 	return engine->set_default_submission ==  in intel_engine_in_execlists_submission_mode()