Lines Matching full:engine
49 * shouldn't we just need a set of those per engine command streamer? This is
51 * rings, the engine cs shifts to a new "ring buffer" with every context
66 * Now that ringbuffers belong per-context (and not per-engine, like before)
67 * and that contexts are uniquely tied to a given engine (and not reusable,
70 * - One ringbuffer per-engine inside each context.
71 * - One backing object per-engine inside each context.
75 * more complex, because we don't know at creation time which engine is going
80 * gets populated for a given engine once we receive an execbuffer. If later
82 * engine, we allocate/populate a new ringbuffer and context backing object and
99 * for the appropriate engine: this structure contains a copy of the context's
103 * If the engine's request queue was empty before the request was added, the
188 * We allow only a single request through the virtual engine at a time
193 * scheduling -- each real engine takes the next available request
200 * engine, sorted by priority. Here we preallocate the nodes we need
201 * for the virtual engine, indexed by physical_engine->id.
211 * If we receive a submit-fence from a master engine, we will only
220 /* And finally, which physical engines this virtual engine maps onto. */
225 static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine) in to_virtual_engine() argument
227 GEM_BUG_ON(!intel_engine_is_virtual(engine)); in to_virtual_engine()
228 return container_of(engine, struct virtual_engine, base); in to_virtual_engine()
232 struct intel_engine_cs *engine);
236 const struct intel_engine_cs *engine,
241 const struct intel_engine_cs *engine,
244 static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) in lrc_ring_mi_mode() argument
246 if (INTEL_GEN(engine->i915) >= 12) in lrc_ring_mi_mode()
248 else if (INTEL_GEN(engine->i915) >= 9) in lrc_ring_mi_mode()
250 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
256 static int lrc_ring_gpr0(const struct intel_engine_cs *engine) in lrc_ring_gpr0() argument
258 if (INTEL_GEN(engine->i915) >= 12) in lrc_ring_gpr0()
260 else if (INTEL_GEN(engine->i915) >= 9) in lrc_ring_gpr0()
262 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
268 static int lrc_ring_wa_bb_per_ctx(const struct intel_engine_cs *engine) in lrc_ring_wa_bb_per_ctx() argument
270 if (INTEL_GEN(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
272 else if (INTEL_GEN(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
278 static int lrc_ring_indirect_ptr(const struct intel_engine_cs *engine) in lrc_ring_indirect_ptr() argument
282 x = lrc_ring_wa_bb_per_ctx(engine); in lrc_ring_indirect_ptr()
289 static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine) in lrc_ring_indirect_offset() argument
293 x = lrc_ring_indirect_ptr(engine); in lrc_ring_indirect_offset()
300 static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine) in lrc_ring_cmd_buf_cctl() argument
302 if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
305 if (INTEL_GEN(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
307 else if (INTEL_GEN(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
314 lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine) in lrc_ring_indirect_offset_default() argument
316 switch (INTEL_GEN(engine->i915)) { in lrc_ring_indirect_offset_default()
318 MISSING_CASE(INTEL_GEN(engine->i915)); in lrc_ring_indirect_offset_default()
335 const struct intel_engine_cs *engine, in lrc_ring_setup_indirect_ctx() argument
341 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_ring_setup_indirect_ctx()
342 regs[lrc_ring_indirect_ptr(engine) + 1] = in lrc_ring_setup_indirect_ctx()
345 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_ring_setup_indirect_ctx()
346 regs[lrc_ring_indirect_offset(engine) + 1] = in lrc_ring_setup_indirect_ctx()
347 lrc_ring_indirect_offset_default(engine) << 6; in lrc_ring_setup_indirect_ctx()
389 static inline u32 intel_hws_preempt_address(struct intel_engine_cs *engine) in intel_hws_preempt_address() argument
391 return (i915_ggtt_offset(engine->status_page.vma) + in intel_hws_preempt_address()
396 ring_set_paused(const struct intel_engine_cs *engine, int state) in ring_set_paused() argument
400 * engine->emit_fini_breadcrumb. If the dword is true, in ring_set_paused()
404 engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state; in ring_set_paused()
457 static inline bool need_preempt(const struct intel_engine_cs *engine, in need_preempt() argument
463 if (!intel_engine_has_semaphores(engine)) in need_preempt()
485 if (engine->execlists.queue_priority_hint <= last_prio) in need_preempt()
492 if (!list_is_last(&rq->sched.link, &engine->active.requests) && in need_preempt()
498 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in need_preempt()
501 if (engine == ve->siblings[0]) { /* only preempt one sibling */ in need_preempt()
525 return queue_prio(&engine->execlists) > last_prio; in need_preempt()
563 * bits 48:53: engine instance
566 * bits 61-63: engine class
568 * engine info, SW context ID and SW counter need to form a unique number
572 lrc_descriptor(struct intel_context *ce, struct intel_engine_cs *engine) in lrc_descriptor() argument
582 if (IS_GEN(engine->i915, 8)) in lrc_descriptor()
595 const struct intel_engine_cs *engine, in set_offsets() argument
606 const u32 base = engine->mmio_base; in set_offsets()
626 if (INTEL_GEN(engine->i915) >= 11) in set_offsets()
657 if (INTEL_GEN(engine->i915) >= 10) in set_offsets()
1077 static const u8 *reg_offsets(const struct intel_engine_cs *engine) in reg_offsets() argument
1083 * physical engines for virtual engine. in reg_offsets()
1085 GEM_BUG_ON(INTEL_GEN(engine->i915) >= 12 && in reg_offsets()
1086 !intel_engine_has_relative_mmio(engine)); in reg_offsets()
1088 if (engine->class == RENDER_CLASS) { in reg_offsets()
1089 if (INTEL_GEN(engine->i915) >= 12) in reg_offsets()
1091 else if (INTEL_GEN(engine->i915) >= 11) in reg_offsets()
1093 else if (INTEL_GEN(engine->i915) >= 9) in reg_offsets()
1098 if (INTEL_GEN(engine->i915) >= 12) in reg_offsets()
1100 else if (INTEL_GEN(engine->i915) >= 9) in reg_offsets()
1108 __unwind_incomplete_requests(struct intel_engine_cs *engine) in __unwind_incomplete_requests() argument
1114 lockdep_assert_held(&engine->active.lock); in __unwind_incomplete_requests()
1117 &engine->active.requests, in __unwind_incomplete_requests()
1126 * If this request is not native to this physical engine (i.e. in __unwind_incomplete_requests()
1128 * engine so that it can be moved across onto another physical in __unwind_incomplete_requests()
1129 * engine as load dictates. in __unwind_incomplete_requests()
1131 if (likely(rq->execution_mask == engine->mask)) { in __unwind_incomplete_requests()
1135 pl = i915_sched_lookup_priolist(engine, prio); in __unwind_incomplete_requests()
1137 GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); in __unwind_incomplete_requests()
1150 struct intel_engine_cs *owner = rq->context->engine; in __unwind_incomplete_requests()
1152 WRITE_ONCE(rq->engine, owner); in __unwind_incomplete_requests()
1164 struct intel_engine_cs *engine = in execlists_unwind_incomplete_requests() local
1165 container_of(execlists, typeof(*engine), execlists); in execlists_unwind_incomplete_requests()
1167 return __unwind_incomplete_requests(engine); in execlists_unwind_incomplete_requests()
1180 atomic_notifier_call_chain(&rq->engine->context_status_notifier, in execlists_context_status_change()
1184 static void intel_engine_context_in(struct intel_engine_cs *engine) in intel_engine_context_in() argument
1188 if (atomic_add_unless(&engine->stats.active, 1, 0)) in intel_engine_context_in()
1191 write_seqlock_irqsave(&engine->stats.lock, flags); in intel_engine_context_in()
1192 if (!atomic_add_unless(&engine->stats.active, 1, 0)) { in intel_engine_context_in()
1193 engine->stats.start = ktime_get(); in intel_engine_context_in()
1194 atomic_inc(&engine->stats.active); in intel_engine_context_in()
1196 write_sequnlock_irqrestore(&engine->stats.lock, flags); in intel_engine_context_in()
1199 static void intel_engine_context_out(struct intel_engine_cs *engine) in intel_engine_context_out() argument
1203 GEM_BUG_ON(!atomic_read(&engine->stats.active)); in intel_engine_context_out()
1205 if (atomic_add_unless(&engine->stats.active, -1, 1)) in intel_engine_context_out()
1208 write_seqlock_irqsave(&engine->stats.lock, flags); in intel_engine_context_out()
1209 if (atomic_dec_and_test(&engine->stats.active)) { in intel_engine_context_out()
1210 engine->stats.total = in intel_engine_context_out()
1211 ktime_add(engine->stats.total, in intel_engine_context_out()
1212 ktime_sub(ktime_get(), engine->stats.start)); in intel_engine_context_out()
1214 write_sequnlock_irqrestore(&engine->stats.lock, flags); in intel_engine_context_out()
1219 const struct intel_engine_cs *engine) in execlists_check_context() argument
1228 engine->name, in execlists_check_context()
1238 engine->name, in execlists_check_context()
1245 x = lrc_ring_mi_mode(engine); in execlists_check_context()
1248 engine->name, regs[x + 1]); in execlists_check_context()
1258 struct intel_engine_cs *engine) in restore_default_state() argument
1262 regs = memset(ce->lrc_reg_state, 0, engine->context_size - PAGE_SIZE); in restore_default_state()
1263 execlists_init_reg_state(regs, ce, engine, ce->ring, true); in restore_default_state()
1269 struct intel_engine_cs *engine) in reset_active() argument
1289 ENGINE_TRACE(engine, "{ rq=%llx:%lld }\n", in reset_active()
1300 restore_default_state(ce, engine); in reset_active()
1301 __execlists_update_reg_state(ce, engine, head); in reset_active()
1341 struct intel_engine_cs * const engine = rq->engine; in __execlists_schedule_in() local
1347 reset_active(rq, engine); in __execlists_schedule_in()
1350 execlists_check_context(ce, engine); in __execlists_schedule_in()
1358 unsigned int tag = ffs(READ_ONCE(engine->context_tag)); in __execlists_schedule_in()
1361 clear_bit(tag - 1, &engine->context_tag); in __execlists_schedule_in()
1367 ce->lrc.ccid |= engine->execlists.ccid; in __execlists_schedule_in()
1369 __intel_gt_pm_get(engine->gt); in __execlists_schedule_in()
1370 if (engine->fw_domain && !atomic_fetch_inc(&engine->fw_active)) in __execlists_schedule_in()
1371 intel_uncore_forcewake_get(engine->uncore, engine->fw_domain); in __execlists_schedule_in()
1373 intel_engine_context_in(engine); in __execlists_schedule_in()
1375 return engine; in __execlists_schedule_in()
1384 GEM_BUG_ON(!intel_engine_pm_is_awake(rq->engine)); in execlists_schedule_in()
1395 GEM_BUG_ON(intel_context_inflight(ce) != rq->engine); in execlists_schedule_in()
1410 struct intel_engine_cs * const engine, in __execlists_schedule_out() argument
1416 * NB process_csb() is not under the engine->active.lock and hence in __execlists_schedule_out()
1422 * If we have just completed this context, the engine may now be in __execlists_schedule_out()
1427 intel_engine_add_retire(engine, ce->timeline); in __execlists_schedule_out()
1433 GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag)); in __execlists_schedule_out()
1434 set_bit(ccid - 1, &engine->context_tag); in __execlists_schedule_out()
1438 intel_engine_context_out(engine); in __execlists_schedule_out()
1440 if (engine->fw_domain && !atomic_dec_return(&engine->fw_active)) in __execlists_schedule_out()
1441 intel_uncore_forcewake_put(engine->uncore, engine->fw_domain); in __execlists_schedule_out()
1442 intel_gt_pm_put_async(engine->gt); in __execlists_schedule_out()
1445 * If this is part of a virtual engine, its next request may in __execlists_schedule_out()
1449 * engine). Hopefully, we will already have submitted the next in __execlists_schedule_out()
1453 if (ce->engine != engine) in __execlists_schedule_out()
1560 const struct intel_engine_cs *engine = in trace_ports() local
1561 container_of(execlists, typeof(*engine), execlists); in trace_ports()
1567 ENGINE_TRACE(engine, "%s { %s%s }\n", msg, in trace_ports()
1582 struct intel_engine_cs *engine = in assert_pending_valid() local
1583 container_of(execlists, typeof(*engine), execlists); in assert_pending_valid()
1597 engine->name); in assert_pending_valid()
1603 engine->name, execlists_num_ports(execlists)); in assert_pending_valid()
1616 engine->name, in assert_pending_valid()
1625 engine->name, in assert_pending_valid()
1639 engine->name, in assert_pending_valid()
1656 engine->name, in assert_pending_valid()
1665 engine->name, in assert_pending_valid()
1674 engine->name, in assert_pending_valid()
1690 static void execlists_submit_ports(struct intel_engine_cs *engine) in execlists_submit_ports() argument
1692 struct intel_engine_execlists *execlists = &engine->execlists; in execlists_submit_ports()
1705 GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); in execlists_submit_ports()
1779 struct intel_engine_cs *engine) in virtual_update_register_offsets() argument
1781 set_offsets(regs, reg_offsets(engine), engine, false); in virtual_update_register_offsets()
1786 const struct intel_engine_cs *engine) in virtual_matches() argument
1790 if (!(rq->execution_mask & engine->mask)) /* We peeked too soon! */ in virtual_matches()
1797 * then. This restricts us to only using the active engine in virtual_matches()
1803 if (inflight && inflight != engine) in virtual_matches()
1810 struct intel_engine_cs *engine) in virtual_xfer_context() argument
1814 if (likely(engine == ve->siblings[0])) in virtual_xfer_context()
1818 if (!intel_engine_has_relative_mmio(engine)) in virtual_xfer_context()
1820 engine); in virtual_xfer_context()
1823 * Move the bound engine to the top of the list for in virtual_xfer_context()
1829 if (ve->siblings[n] == engine) { in virtual_xfer_context()
1871 if (w->engine != rq->engine) in defer_request()
1894 static void defer_active(struct intel_engine_cs *engine) in defer_active() argument
1898 rq = __unwind_incomplete_requests(engine); in defer_active()
1902 defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq))); in defer_active()
1906 need_timeslice(const struct intel_engine_cs *engine, in need_timeslice() argument
1912 if (!intel_engine_has_timeslices(engine)) in need_timeslice()
1915 hint = engine->execlists.queue_priority_hint; in need_timeslice()
1919 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in need_timeslice()
1923 if (!inflight || inflight == engine) { in need_timeslice()
1934 if (!list_is_last(&rq->sched.link, &engine->active.requests)) in need_timeslice()
1968 switch_prio(struct intel_engine_cs *engine, const struct i915_request *rq) in switch_prio() argument
1970 if (list_is_last(&rq->sched.link, &engine->active.requests)) in switch_prio()
1971 return engine->execlists.queue_priority_hint; in switch_prio()
1977 timeslice(const struct intel_engine_cs *engine) in timeslice() argument
1979 return READ_ONCE(engine->props.timeslice_duration_ms); in timeslice()
1982 static unsigned long active_timeslice(const struct intel_engine_cs *engine) in active_timeslice() argument
1984 const struct intel_engine_execlists *execlists = &engine->execlists; in active_timeslice()
1993 return timeslice(engine); in active_timeslice()
1996 static void set_timeslice(struct intel_engine_cs *engine) in set_timeslice() argument
2000 if (!intel_engine_has_timeslices(engine)) in set_timeslice()
2003 duration = active_timeslice(engine); in set_timeslice()
2004 ENGINE_TRACE(engine, "bump timeslicing, interval:%lu", duration); in set_timeslice()
2006 set_timer_ms(&engine->execlists.timer, duration); in set_timeslice()
2009 static void start_timeslice(struct intel_engine_cs *engine, int prio) in start_timeslice() argument
2011 struct intel_engine_execlists *execlists = &engine->execlists; in start_timeslice()
2014 if (!intel_engine_has_timeslices(engine)) in start_timeslice()
2024 duration = timeslice(engine); in start_timeslice()
2025 ENGINE_TRACE(engine, in start_timeslice()
2037 static unsigned long active_preempt_timeout(struct intel_engine_cs *engine, in active_preempt_timeout() argument
2047 return READ_ONCE(engine->props.preempt_timeout_ms); in active_preempt_timeout()
2050 static void set_preempt_timeout(struct intel_engine_cs *engine, in set_preempt_timeout() argument
2053 if (!intel_engine_has_preempt_reset(engine)) in set_preempt_timeout()
2056 set_timer_ms(&engine->execlists.preempt, in set_preempt_timeout()
2057 active_preempt_timeout(engine, rq)); in set_preempt_timeout()
2073 static void execlists_dequeue(struct intel_engine_cs *engine) in execlists_dequeue() argument
2075 struct intel_engine_execlists * const execlists = &engine->execlists; in execlists_dequeue()
2107 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_dequeue()
2110 if (!rq) { /* lazily cleanup after another engine handled rq */ in execlists_dequeue()
2117 if (!virtual_matches(ve, rq, engine)) { in execlists_dequeue()
2148 if (need_preempt(engine, last, rb)) { in execlists_dequeue()
2154 ENGINE_TRACE(engine, in execlists_dequeue()
2167 ring_set_paused(engine, 1); in execlists_dequeue()
2176 __unwind_incomplete_requests(engine); in execlists_dequeue()
2179 } else if (need_timeslice(engine, last, rb) && in execlists_dequeue()
2186 ENGINE_TRACE(engine, in execlists_dequeue()
2194 ring_set_paused(engine, 1); in execlists_dequeue()
2195 defer_active(engine); in execlists_dequeue()
2222 &engine->active.requests)) { in execlists_dequeue()
2227 start_timeslice(engine, queue_prio(execlists)); in execlists_dequeue()
2235 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_dequeue()
2250 GEM_BUG_ON(rq->engine != &ve->base); in execlists_dequeue()
2254 if (!virtual_matches(ve, rq, engine)) { in execlists_dequeue()
2262 start_timeslice(engine, rq_prio(rq)); in execlists_dequeue()
2266 ENGINE_TRACE(engine, in execlists_dequeue()
2267 "virtual rq=%llx:%lld%s, new engine? %s\n", in execlists_dequeue()
2273 yesno(engine != ve->siblings[0])); in execlists_dequeue()
2281 GEM_BUG_ON(!(rq->execution_mask & engine->mask)); in execlists_dequeue()
2282 WRITE_ONCE(rq->engine, engine); in execlists_dequeue()
2298 virtual_xfer_context(ve, engine); in execlists_dequeue()
2299 GEM_BUG_ON(ve->siblings[0] != engine); in execlists_dequeue()
2307 * Hmm, we have a bunch of virtual engine requests, in execlists_dequeue()
2422 switch_prio(engine, *execlists->pending); in execlists_dequeue()
2439 set_preempt_timeout(engine, *active); in execlists_dequeue()
2440 execlists_submit_ports(engine); in execlists_dequeue()
2442 start_timeslice(engine, execlists->queue_priority_hint); in execlists_dequeue()
2444 ring_set_paused(engine, 0); in execlists_dequeue()
2480 * bits 3-5: engine class
2481 * bits 6-11: engine instance
2545 static void process_csb(struct intel_engine_cs *engine) in process_csb() argument
2547 struct intel_engine_execlists * const execlists = &engine->execlists; in process_csb()
2559 GEM_BUG_ON(!intel_engine_in_execlists_submission_mode(engine)); in process_csb()
2584 * all tracking of what the engine is actually executing. We will in process_csb()
2590 * engine to reset. in process_csb()
2593 ENGINE_TRACE(engine, "cs-irq head=%d, tail=%d\n", head, tail); in process_csb()
2628 ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n", in process_csb()
2633 if (INTEL_GEN(engine->i915) >= 12) in process_csb()
2645 ring_set_paused(engine, 0); in process_csb()
2665 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); in process_csb()
2693 ENGINE_TRACE(engine, in process_csb()
2695 ENGINE_TRACE(engine, in process_csb()
2697 ENGINE_READ(engine, RING_START), in process_csb()
2698 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR, in process_csb()
2699 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR, in process_csb()
2700 ENGINE_READ(engine, RING_CTL), in process_csb()
2701 ENGINE_READ(engine, RING_MI_MODE)); in process_csb()
2702 ENGINE_TRACE(engine, in process_csb()
2709 ENGINE_TRACE(engine, in process_csb()
2723 set_timeslice(engine); in process_csb()
2739 static void __execlists_submission_tasklet(struct intel_engine_cs *const engine) in __execlists_submission_tasklet() argument
2741 lockdep_assert_held(&engine->active.lock); in __execlists_submission_tasklet()
2742 if (!READ_ONCE(engine->execlists.pending[0])) { in __execlists_submission_tasklet()
2744 execlists_dequeue(engine); in __execlists_submission_tasklet()
2760 list_move_tail(&rq->sched.link, &rq->engine->active.hold); in __execlists_hold()
2769 if (w->engine != rq->engine) in __execlists_hold()
2788 static bool execlists_hold(struct intel_engine_cs *engine, in execlists_hold() argument
2794 spin_lock_irq(&engine->active.lock); in execlists_hold()
2801 if (rq->engine != engine) { /* preempted virtual engine */ in execlists_hold()
2802 struct virtual_engine *ve = to_virtual_engine(rq->engine); in execlists_hold()
2811 GEM_BUG_ON(!reset_in_progress(&engine->execlists)); in execlists_hold()
2814 * An unsubmitted request along a virtual engine will in execlists_hold()
2815 * remain on the active (this) engine until we are able in execlists_hold()
2821 GEM_BUG_ON(intel_context_inflight(rq->context) != engine); in execlists_hold()
2827 rq->engine = engine; in execlists_hold()
2837 GEM_BUG_ON(rq->engine != engine); in execlists_hold()
2839 GEM_BUG_ON(list_empty(&engine->active.hold)); in execlists_hold()
2842 spin_unlock_irq(&engine->active.lock); in execlists_hold()
2860 if (s->engine != rq->engine) in hold_request()
2886 i915_sched_lookup_priolist(rq->engine, in __execlists_unhold()
2890 /* Also release any children on this engine that are ready */ in __execlists_unhold()
2899 if (w->engine != rq->engine) in __execlists_unhold()
2916 static void execlists_unhold(struct intel_engine_cs *engine, in execlists_unhold() argument
2919 spin_lock_irq(&engine->active.lock); in execlists_unhold()
2927 if (rq_prio(rq) > engine->execlists.queue_priority_hint) { in execlists_unhold()
2928 engine->execlists.queue_priority_hint = rq_prio(rq); in execlists_unhold()
2929 tasklet_hi_schedule(&engine->execlists.tasklet); in execlists_unhold()
2932 spin_unlock_irq(&engine->active.lock); in execlists_unhold()
2945 struct intel_engine_cs *engine = cap->rq->engine; in execlists_capture_work() local
2950 vma = intel_engine_coredump_add_request(gt->engine, cap->rq, gfp); in execlists_capture_work()
2955 intel_engine_coredump_add_vma(gt->engine, vma, compress); in execlists_capture_work()
2959 gt->simulated = gt->engine->simulated; in execlists_capture_work()
2967 execlists_unhold(engine, cap->rq); in execlists_capture_work()
2973 static struct execlists_capture *capture_regs(struct intel_engine_cs *engine) in capture_regs() argument
2982 cap->error = i915_gpu_coredump_alloc(engine->i915, gfp); in capture_regs()
2986 cap->error->gt = intel_gt_coredump_alloc(engine->gt, gfp); in capture_regs()
2990 cap->error->gt->engine = intel_engine_coredump_alloc(engine, gfp); in capture_regs()
2991 if (!cap->error->gt->engine) in capture_regs()
3006 active_context(struct intel_engine_cs *engine, u32 ccid) in active_context() argument
3008 const struct intel_engine_execlists * const el = &engine->execlists; in active_context()
3019 ENGINE_TRACE(engine, in active_context()
3028 ENGINE_TRACE(engine, in active_context()
3035 ENGINE_TRACE(engine, "ccid:%x not found\n", ccid); in active_context()
3039 static u32 active_ccid(struct intel_engine_cs *engine) in active_ccid() argument
3041 return ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI); in active_ccid()
3044 static void execlists_capture(struct intel_engine_cs *engine) in execlists_capture() argument
3052 * We need to _quickly_ capture the engine state before we reset. in execlists_capture()
3056 cap = capture_regs(engine); in execlists_capture()
3060 spin_lock_irq(&engine->active.lock); in execlists_capture()
3061 cap->rq = active_context(engine, active_ccid(engine)); in execlists_capture()
3066 spin_unlock_irq(&engine->active.lock); in execlists_capture()
3081 * Note that because we have not yet reset the engine at this point, in execlists_capture()
3085 * of that is very low (as capturing of the engine registers should be in execlists_capture()
3090 if (!execlists_hold(engine, cap->rq)) in execlists_capture()
3104 static void execlists_reset(struct intel_engine_cs *engine, const char *msg) in execlists_reset() argument
3106 const unsigned int bit = I915_RESET_ENGINE + engine->id; in execlists_reset()
3107 unsigned long *lock = &engine->gt->reset.flags; in execlists_reset()
3109 if (!intel_has_reset_engine(engine->gt)) in execlists_reset()
3115 ENGINE_TRACE(engine, "reset for %s\n", msg); in execlists_reset()
3118 tasklet_disable_nosync(&engine->execlists.tasklet); in execlists_reset()
3120 ring_set_paused(engine, 1); /* Freeze the current request in place */ in execlists_reset()
3121 execlists_capture(engine); in execlists_reset()
3122 intel_engine_reset(engine, msg); in execlists_reset()
3124 tasklet_enable(&engine->execlists.tasklet); in execlists_reset()
3128 static bool preempt_timeout(const struct intel_engine_cs *const engine) in preempt_timeout() argument
3130 const struct timer_list *t = &engine->execlists.preempt; in preempt_timeout()
3138 return READ_ONCE(engine->execlists.pending[0]); in preempt_timeout()
3147 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; in execlists_submission_tasklet() local
3148 bool timeout = preempt_timeout(engine); in execlists_submission_tasklet()
3150 process_csb(engine); in execlists_submission_tasklet()
3152 if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) { in execlists_submission_tasklet()
3156 if (engine->execlists.error_interrupt & GENMASK(15, 0)) in execlists_submission_tasklet()
3158 else if (engine->execlists.error_interrupt & ERROR_CSB) in execlists_submission_tasklet()
3163 engine->execlists.error_interrupt = 0; in execlists_submission_tasklet()
3164 execlists_reset(engine, msg); in execlists_submission_tasklet()
3167 if (!READ_ONCE(engine->execlists.pending[0]) || timeout) { in execlists_submission_tasklet()
3170 spin_lock_irqsave(&engine->active.lock, flags); in execlists_submission_tasklet()
3171 __execlists_submission_tasklet(engine); in execlists_submission_tasklet()
3172 spin_unlock_irqrestore(&engine->active.lock, flags); in execlists_submission_tasklet()
3175 if (unlikely(timeout && preempt_timeout(engine))) { in execlists_submission_tasklet()
3176 cancel_timer(&engine->execlists.preempt); in execlists_submission_tasklet()
3177 execlists_reset(engine, "preemption time out"); in execlists_submission_tasklet()
3201 static void queue_request(struct intel_engine_cs *engine, in queue_request() argument
3206 i915_sched_lookup_priolist(engine, rq_prio(rq))); in queue_request()
3210 static void __submit_queue_imm(struct intel_engine_cs *engine) in __submit_queue_imm() argument
3212 struct intel_engine_execlists * const execlists = &engine->execlists; in __submit_queue_imm()
3215 return; /* defer until we restart the engine following reset */ in __submit_queue_imm()
3217 __execlists_submission_tasklet(engine); in __submit_queue_imm()
3220 static void submit_queue(struct intel_engine_cs *engine, in submit_queue() argument
3223 struct intel_engine_execlists *execlists = &engine->execlists; in submit_queue()
3229 __submit_queue_imm(engine); in submit_queue()
3232 static bool ancestor_on_hold(const struct intel_engine_cs *engine, in ancestor_on_hold() argument
3236 return !list_empty(&engine->active.hold) && hold_request(rq); in ancestor_on_hold()
3239 static void flush_csb(struct intel_engine_cs *engine) in flush_csb() argument
3241 struct intel_engine_execlists *el = &engine->execlists; in flush_csb()
3245 process_csb(engine); in flush_csb()
3252 struct intel_engine_cs *engine = request->engine; in execlists_submit_request() local
3256 flush_csb(engine); in execlists_submit_request()
3259 spin_lock_irqsave(&engine->active.lock, flags); in execlists_submit_request()
3261 if (unlikely(ancestor_on_hold(engine, request))) { in execlists_submit_request()
3263 list_add_tail(&request->sched.link, &engine->active.hold); in execlists_submit_request()
3266 queue_request(engine, request); in execlists_submit_request()
3268 GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); in execlists_submit_request()
3271 submit_queue(engine, request); in execlists_submit_request()
3274 spin_unlock_irqrestore(&engine->active.lock, flags); in execlists_submit_request()
3298 set_redzone(void *vaddr, const struct intel_engine_cs *engine) in set_redzone() argument
3303 vaddr += engine->context_size; in set_redzone()
3309 check_redzone(const void *vaddr, const struct intel_engine_cs *engine) in check_redzone() argument
3314 vaddr += engine->context_size; in check_redzone()
3317 drm_err_once(&engine->i915->drm, in check_redzone()
3319 engine->name); in check_redzone()
3325 ce->engine); in execlists_context_unpin()
3362 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
3369 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
3378 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
3385 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
3436 const struct intel_engine_cs *engine, in setup_indirect_ctx_bb() argument
3447 lrc_ring_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
3455 const struct intel_engine_cs *engine, in __execlists_update_reg_state() argument
3470 if (engine->class == RENDER_CLASS) { in __execlists_update_reg_state()
3472 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in __execlists_update_reg_state()
3474 i915_oa_init_reg_state(ce, engine); in __execlists_update_reg_state()
3481 if (ce->engine->class == RENDER_CLASS) in __execlists_update_reg_state()
3485 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in __execlists_update_reg_state()
3486 setup_indirect_ctx_bb(ce, engine, fn); in __execlists_update_reg_state()
3498 i915_coherent_map_type(ce->engine->i915) | in execlists_context_pre_pin()
3506 struct intel_engine_cs *engine, in __execlists_context_pin() argument
3509 ce->lrc.lrca = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE; in __execlists_context_pin()
3511 __execlists_update_reg_state(ce, engine, ce->ring->tail); in __execlists_context_pin()
3518 return __execlists_context_pin(ce, ce->engine, vaddr); in execlists_context_pin()
3523 return __execlists_context_alloc(ce, ce->engine); in execlists_context_alloc()
3535 ce, ce->engine, ce->ring, true); in execlists_context_reset()
3536 __execlists_update_reg_state(ce, ce->engine, ce->ring->tail); in execlists_context_reset()
3607 const struct intel_engine_cs * const engine = rq->engine; in emit_pdps() local
3612 GEM_BUG_ON(intel_vgpu_active(rq->engine->i915)); in emit_pdps()
3622 err = engine->emit_flush(rq, EMIT_FLUSH); in emit_pdps()
3627 err = engine->emit_flush(rq, EMIT_INVALIDATE); in emit_pdps()
3639 u32 base = engine->mmio_base; in emit_pdps()
3669 * state of engine initialisation and liveness of the in execlists_request_alloc()
3681 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); in execlists_request_alloc()
3706 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch) in gen8_emit_flush_coherentl3_wa() argument
3711 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
3726 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
3748 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) in gen8_init_indirectctx_bb() argument
3754 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
3755 batch = gen8_emit_flush_coherentl3_wa(engine, batch); in gen8_init_indirectctx_bb()
3800 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) in gen9_init_indirectctx_bb() argument
3828 batch = gen8_emit_flush_coherentl3_wa(engine, batch); in gen9_init_indirectctx_bb()
3841 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
3873 gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) in gen10_init_indirectctx_bb() argument
3880 * Ensure the engine is idle prior to programming a in gen10_init_indirectctx_bb()
3908 static int lrc_setup_wa_ctx(struct intel_engine_cs *engine) in lrc_setup_wa_ctx() argument
3914 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_OBJ_SIZE); in lrc_setup_wa_ctx()
3918 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_setup_wa_ctx()
3928 engine->wa_ctx.vma = vma; in lrc_setup_wa_ctx()
3936 static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine) in lrc_destroy_wa_ctx() argument
3938 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_destroy_wa_ctx()
3941 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
3943 static int intel_init_workaround_bb(struct intel_engine_cs *engine) in intel_init_workaround_bb() argument
3945 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in intel_init_workaround_bb()
3953 if (engine->class != RENDER_CLASS) in intel_init_workaround_bb()
3956 switch (INTEL_GEN(engine->i915)) { in intel_init_workaround_bb()
3973 MISSING_CASE(INTEL_GEN(engine->i915)); in intel_init_workaround_bb()
3977 ret = lrc_setup_wa_ctx(engine); in intel_init_workaround_bb()
3979 drm_dbg(&engine->i915->drm, in intel_init_workaround_bb()
4000 batch_ptr = wa_bb_fn[i](engine, batch_ptr); in intel_init_workaround_bb()
4008 lrc_destroy_wa_ctx(engine); in intel_init_workaround_bb()
4013 static void reset_csb_pointers(struct intel_engine_cs *engine) in reset_csb_pointers() argument
4015 struct intel_engine_execlists * const execlists = &engine->execlists; in reset_csb_pointers()
4018 ring_set_paused(engine, 0); in reset_csb_pointers()
4024 ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR, in reset_csb_pointers()
4026 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); in reset_csb_pointers()
4047 ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR, in reset_csb_pointers()
4049 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); in reset_csb_pointers()
4054 static void execlists_sanitize(struct intel_engine_cs *engine) in execlists_sanitize() argument
4066 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE); in execlists_sanitize()
4068 reset_csb_pointers(engine); in execlists_sanitize()
4075 intel_timeline_reset_seqno(engine->kernel_context->timeline); in execlists_sanitize()
4078 clflush_cache_range(engine->status_page.addr, PAGE_SIZE); in execlists_sanitize()
4081 static void enable_error_interrupt(struct intel_engine_cs *engine) in enable_error_interrupt() argument
4085 engine->execlists.error_interrupt = 0; in enable_error_interrupt()
4086 ENGINE_WRITE(engine, RING_EMR, ~0u); in enable_error_interrupt()
4087 ENGINE_WRITE(engine, RING_EIR, ~0u); /* clear all existing errors */ in enable_error_interrupt()
4089 status = ENGINE_READ(engine, RING_ESR); in enable_error_interrupt()
4091 drm_err(&engine->i915->drm, in enable_error_interrupt()
4092 "engine '%s' resumed still in error: %08x\n", in enable_error_interrupt()
4093 engine->name, status); in enable_error_interrupt()
4094 __intel_gt_reset(engine->gt, engine->mask); in enable_error_interrupt()
4121 ENGINE_WRITE(engine, RING_EMR, ~I915_ERROR_INSTRUCTION); in enable_error_interrupt()
4124 static void enable_execlists(struct intel_engine_cs *engine) in enable_execlists() argument
4128 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL); in enable_execlists()
4130 intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */ in enable_execlists()
4132 if (INTEL_GEN(engine->i915) >= 11) in enable_execlists()
4136 ENGINE_WRITE_FW(engine, RING_MODE_GEN7, mode); in enable_execlists()
4138 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); in enable_execlists()
4140 ENGINE_WRITE_FW(engine, in enable_execlists()
4142 i915_ggtt_offset(engine->status_page.vma)); in enable_execlists()
4143 ENGINE_POSTING_READ(engine, RING_HWS_PGA); in enable_execlists()
4145 enable_error_interrupt(engine); in enable_execlists()
4147 engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0); in enable_execlists()
4150 static bool unexpected_starting_state(struct intel_engine_cs *engine) in unexpected_starting_state() argument
4154 if (ENGINE_READ_FW(engine, RING_MI_MODE) & STOP_RING) { in unexpected_starting_state()
4155 drm_dbg(&engine->i915->drm, in unexpected_starting_state()
4163 static int execlists_resume(struct intel_engine_cs *engine) in execlists_resume() argument
4165 intel_mocs_init_engine(engine); in execlists_resume()
4167 intel_breadcrumbs_reset(engine->breadcrumbs); in execlists_resume()
4169 if (GEM_SHOW_DEBUG() && unexpected_starting_state(engine)) { in execlists_resume()
4172 intel_engine_dump(engine, &p, NULL); in execlists_resume()
4175 enable_execlists(engine); in execlists_resume()
4180 static void execlists_reset_prepare(struct intel_engine_cs *engine) in execlists_reset_prepare() argument
4182 struct intel_engine_execlists * const execlists = &engine->execlists; in execlists_reset_prepare()
4185 ENGINE_TRACE(engine, "depth<-%d\n", in execlists_reset_prepare()
4191 * is completed by one engine, it may then queue a request in execlists_reset_prepare()
4193 * calling engine->resume() and also writing the ELSP. in execlists_reset_prepare()
4201 spin_lock_irqsave(&engine->active.lock, flags); in execlists_reset_prepare()
4202 spin_unlock_irqrestore(&engine->active.lock, flags); in execlists_reset_prepare()
4216 ring_set_paused(engine, 1); in execlists_reset_prepare()
4217 intel_engine_stop_cs(engine); in execlists_reset_prepare()
4219 engine->execlists.reset_ccid = active_ccid(engine); in execlists_reset_prepare()
4222 static void __reset_stop_ring(u32 *regs, const struct intel_engine_cs *engine) in __reset_stop_ring() argument
4226 x = lrc_ring_mi_mode(engine); in __reset_stop_ring()
4234 const struct intel_engine_cs *engine) in __execlists_reset_reg_state() argument
4238 __reset_stop_ring(regs, engine); in __execlists_reset_reg_state()
4241 static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) in __execlists_reset() argument
4243 struct intel_engine_execlists * const execlists = &engine->execlists; in __execlists_reset()
4252 process_csb(engine); /* drain preemption events */ in __execlists_reset()
4255 reset_csb_pointers(engine); in __execlists_reset()
4262 rq = active_context(engine, engine->execlists.reset_ccid); in __execlists_reset()
4275 /* We still have requests in-flight; the engine should be active */ in __execlists_reset()
4276 GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); in __execlists_reset()
4322 ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n", in __execlists_reset()
4324 __execlists_reset_reg_state(ce, engine); in __execlists_reset()
4325 __execlists_update_reg_state(ce, engine, head); in __execlists_reset()
4331 __unwind_incomplete_requests(engine); in __execlists_reset()
4334 static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled) in execlists_reset_rewind() argument
4338 ENGINE_TRACE(engine, "\n"); in execlists_reset_rewind()
4340 spin_lock_irqsave(&engine->active.lock, flags); in execlists_reset_rewind()
4342 __execlists_reset(engine, stalled); in execlists_reset_rewind()
4344 spin_unlock_irqrestore(&engine->active.lock, flags); in execlists_reset_rewind()
4349 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; in nop_submission_tasklet() local
4352 WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN); in nop_submission_tasklet()
4355 static void execlists_reset_cancel(struct intel_engine_cs *engine) in execlists_reset_cancel() argument
4357 struct intel_engine_execlists * const execlists = &engine->execlists; in execlists_reset_cancel()
4362 ENGINE_TRACE(engine, "\n"); in execlists_reset_cancel()
4365 * Before we call engine->cancel_requests(), we should have exclusive in execlists_reset_cancel()
4378 spin_lock_irqsave(&engine->active.lock, flags); in execlists_reset_cancel()
4380 __execlists_reset(engine, true); in execlists_reset_cancel()
4383 list_for_each_entry(rq, &engine->active.requests, sched.link) in execlists_reset_cancel()
4401 list_for_each_entry(rq, &engine->active.hold, sched.link) in execlists_reset_cancel()
4407 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_reset_cancel()
4417 rq->engine = engine; in execlists_reset_cancel()
4434 spin_unlock_irqrestore(&engine->active.lock, flags); in execlists_reset_cancel()
4437 static void execlists_reset_finish(struct intel_engine_cs *engine) in execlists_reset_finish() argument
4439 struct intel_engine_execlists * const execlists = &engine->execlists; in execlists_reset_finish()
4453 ENGINE_TRACE(engine, "depth->%d\n", in execlists_reset_finish()
4478 * (engine->emit_fini_breadcrumb). in gen8_emit_bb_start_noarb()
4518 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) in gen8_logical_ring_enable_irq() argument
4520 ENGINE_WRITE(engine, RING_IMR, in gen8_logical_ring_enable_irq()
4521 ~(engine->irq_enable_mask | engine->irq_keep_mask)); in gen8_logical_ring_enable_irq()
4522 ENGINE_POSTING_READ(engine, RING_IMR); in gen8_logical_ring_enable_irq()
4525 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine) in gen8_logical_ring_disable_irq() argument
4527 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask); in gen8_logical_ring_disable_irq()
4549 if (request->engine->class == VIDEO_DECODE_CLASS) in gen8_emit_flush()
4592 if (IS_GEN(request->engine->i915, 9)) in gen8_emit_flush_render()
4596 if (IS_KBL_GT_REVID(request->engine->i915, 0, KBL_REVID_B0)) in gen8_emit_flush_render()
4686 static i915_reg_t aux_inv_reg(const struct intel_engine_cs *engine) in aux_inv_reg() argument
4700 if (engine->class == VIDEO_DECODE_CLASS) in aux_inv_reg()
4701 return vd[engine->instance]; in aux_inv_reg()
4703 if (engine->class == VIDEO_ENHANCEMENT_CLASS) in aux_inv_reg()
4704 return ve[engine->instance]; in aux_inv_reg()
4802 aux_inv = request->engine->mask & ~BIT(BCS0); in gen12_emit_flush()
4824 if (request->engine->class == VIDEO_DECODE_CLASS) in gen12_emit_flush()
4834 struct intel_engine_cs *engine; in gen12_emit_flush() local
4838 for_each_engine_masked(engine, request->engine->gt, in gen12_emit_flush()
4840 *cs++ = i915_mmio_reg_offset(aux_inv_reg(engine)); in gen12_emit_flush()
4887 *cs++ = intel_hws_preempt_address(request->engine); in emit_preempt_busywait()
4899 if (intel_engine_has_semaphores(request->engine)) in gen8_emit_fini_breadcrumb_tail()
4978 *cs++ = intel_hws_preempt_address(request->engine); in gen12_emit_preempt_busywait()
4992 if (intel_engine_has_semaphores(request->engine)) in gen12_emit_fini_breadcrumb_tail()
5028 static void execlists_park(struct intel_engine_cs *engine) in execlists_park() argument
5030 cancel_timer(&engine->execlists.timer); in execlists_park()
5031 cancel_timer(&engine->execlists.preempt); in execlists_park()
5034 void intel_execlists_set_default_submission(struct intel_engine_cs *engine) in intel_execlists_set_default_submission() argument
5036 engine->submit_request = execlists_submit_request; in intel_execlists_set_default_submission()
5037 engine->schedule = i915_schedule; in intel_execlists_set_default_submission()
5038 engine->execlists.tasklet.func = execlists_submission_tasklet; in intel_execlists_set_default_submission()
5040 engine->reset.prepare = execlists_reset_prepare; in intel_execlists_set_default_submission()
5041 engine->reset.rewind = execlists_reset_rewind; in intel_execlists_set_default_submission()
5042 engine->reset.cancel = execlists_reset_cancel; in intel_execlists_set_default_submission()
5043 engine->reset.finish = execlists_reset_finish; in intel_execlists_set_default_submission()
5045 engine->park = execlists_park; in intel_execlists_set_default_submission()
5046 engine->unpark = NULL; in intel_execlists_set_default_submission()
5048 engine->flags |= I915_ENGINE_SUPPORTS_STATS; in intel_execlists_set_default_submission()
5049 if (!intel_vgpu_active(engine->i915)) { in intel_execlists_set_default_submission()
5050 engine->flags |= I915_ENGINE_HAS_SEMAPHORES; in intel_execlists_set_default_submission()
5051 if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) { in intel_execlists_set_default_submission()
5052 engine->flags |= I915_ENGINE_HAS_PREEMPTION; in intel_execlists_set_default_submission()
5054 engine->flags |= I915_ENGINE_HAS_TIMESLICES; in intel_execlists_set_default_submission()
5058 if (INTEL_GEN(engine->i915) >= 12) in intel_execlists_set_default_submission()
5059 engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO; in intel_execlists_set_default_submission()
5061 if (intel_engine_has_preemption(engine)) in intel_execlists_set_default_submission()
5062 engine->emit_bb_start = gen8_emit_bb_start; in intel_execlists_set_default_submission()
5064 engine->emit_bb_start = gen8_emit_bb_start_noarb; in intel_execlists_set_default_submission()
5067 static void execlists_shutdown(struct intel_engine_cs *engine) in execlists_shutdown() argument
5070 del_timer_sync(&engine->execlists.timer); in execlists_shutdown()
5071 del_timer_sync(&engine->execlists.preempt); in execlists_shutdown()
5072 tasklet_kill(&engine->execlists.tasklet); in execlists_shutdown()
5075 static void execlists_release(struct intel_engine_cs *engine) in execlists_release() argument
5077 engine->sanitize = NULL; /* no longer in control, nothing to sanitize */ in execlists_release()
5079 execlists_shutdown(engine); in execlists_release()
5081 intel_engine_cleanup_common(engine); in execlists_release()
5082 lrc_destroy_wa_ctx(engine); in execlists_release()
5086 logical_ring_default_vfuncs(struct intel_engine_cs *engine) in logical_ring_default_vfuncs() argument
5088 /* Default vfuncs which can be overriden by each engine. */ in logical_ring_default_vfuncs()
5090 engine->resume = execlists_resume; in logical_ring_default_vfuncs()
5092 engine->cops = &execlists_context_ops; in logical_ring_default_vfuncs()
5093 engine->request_alloc = execlists_request_alloc; in logical_ring_default_vfuncs()
5095 engine->emit_flush = gen8_emit_flush; in logical_ring_default_vfuncs()
5096 engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; in logical_ring_default_vfuncs()
5097 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb; in logical_ring_default_vfuncs()
5098 if (INTEL_GEN(engine->i915) >= 12) { in logical_ring_default_vfuncs()
5099 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb; in logical_ring_default_vfuncs()
5100 engine->emit_flush = gen12_emit_flush; in logical_ring_default_vfuncs()
5102 engine->set_default_submission = intel_execlists_set_default_submission; in logical_ring_default_vfuncs()
5104 if (INTEL_GEN(engine->i915) < 11) { in logical_ring_default_vfuncs()
5105 engine->irq_enable = gen8_logical_ring_enable_irq; in logical_ring_default_vfuncs()
5106 engine->irq_disable = gen8_logical_ring_disable_irq; in logical_ring_default_vfuncs()
5118 logical_ring_default_irqs(struct intel_engine_cs *engine) in logical_ring_default_irqs() argument
5122 if (INTEL_GEN(engine->i915) < 11) { in logical_ring_default_irqs()
5131 shift = irq_shifts[engine->id]; in logical_ring_default_irqs()
5134 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift; in logical_ring_default_irqs()
5135 engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift; in logical_ring_default_irqs()
5136 engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift; in logical_ring_default_irqs()
5137 engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift; in logical_ring_default_irqs()
5140 static void rcs_submission_override(struct intel_engine_cs *engine) in rcs_submission_override() argument
5142 switch (INTEL_GEN(engine->i915)) { in rcs_submission_override()
5144 engine->emit_flush = gen12_emit_flush_render; in rcs_submission_override()
5145 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs; in rcs_submission_override()
5148 engine->emit_flush = gen11_emit_flush_render; in rcs_submission_override()
5149 engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs; in rcs_submission_override()
5152 engine->emit_flush = gen8_emit_flush_render; in rcs_submission_override()
5153 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs; in rcs_submission_override()
5158 int intel_execlists_submission_setup(struct intel_engine_cs *engine) in intel_execlists_submission_setup() argument
5160 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_execlists_submission_setup()
5161 struct drm_i915_private *i915 = engine->i915; in intel_execlists_submission_setup()
5162 struct intel_uncore *uncore = engine->uncore; in intel_execlists_submission_setup()
5163 u32 base = engine->mmio_base; in intel_execlists_submission_setup()
5165 tasklet_init(&engine->execlists.tasklet, in intel_execlists_submission_setup()
5166 execlists_submission_tasklet, (unsigned long)engine); in intel_execlists_submission_setup()
5167 timer_setup(&engine->execlists.timer, execlists_timeslice, 0); in intel_execlists_submission_setup()
5168 timer_setup(&engine->execlists.preempt, execlists_preempt, 0); in intel_execlists_submission_setup()
5170 logical_ring_default_vfuncs(engine); in intel_execlists_submission_setup()
5171 logical_ring_default_irqs(engine); in intel_execlists_submission_setup()
5173 if (engine->class == RENDER_CLASS) in intel_execlists_submission_setup()
5174 rcs_submission_override(engine); in intel_execlists_submission_setup()
5176 if (intel_init_workaround_bb(engine)) in intel_execlists_submission_setup()
5195 (u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; in intel_execlists_submission_setup()
5198 &engine->status_page.addr[intel_hws_csb_write_index(i915)]; in intel_execlists_submission_setup()
5205 if (INTEL_GEN(engine->i915) >= 11) { in intel_execlists_submission_setup()
5206 execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32); in intel_execlists_submission_setup()
5207 execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32); in intel_execlists_submission_setup()
5211 engine->sanitize = execlists_sanitize; in intel_execlists_submission_setup()
5212 engine->release = execlists_release; in intel_execlists_submission_setup()
5218 const struct intel_engine_cs *engine, in init_common_reg_state() argument
5228 if (INTEL_GEN(engine->i915) < 11) in init_common_reg_state()
5238 const struct intel_engine_cs *engine) in init_wa_bb_reg_state() argument
5240 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_reg_state()
5245 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_reg_state()
5246 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = in init_wa_bb_reg_state()
5251 lrc_ring_setup_indirect_ctx(regs, engine, in init_wa_bb_reg_state()
5284 const struct intel_engine_cs *engine, in execlists_init_reg_state() argument
5298 set_offsets(regs, reg_offsets(engine), engine, inhibit); in execlists_init_reg_state()
5300 init_common_reg_state(regs, engine, ring, inhibit); in execlists_init_reg_state()
5303 init_wa_bb_reg_state(regs, engine); in execlists_init_reg_state()
5305 __reset_stop_ring(regs, engine); in execlists_init_reg_state()
5311 struct intel_engine_cs *engine, in populate_lr_context() argument
5319 drm_dbg(&engine->i915->drm, "Could not map object pages!\n"); in populate_lr_context()
5323 set_redzone(vaddr, engine); in populate_lr_context()
5325 if (engine->default_state) { in populate_lr_context()
5326 shmem_read(engine->default_state, 0, in populate_lr_context()
5327 vaddr, engine->context_size); in populate_lr_context()
5340 ce, engine, ring, inhibit); in populate_lr_context()
5342 __i915_gem_object_flush_map(ctx_obj, 0, engine->context_size); in populate_lr_context()
5351 return intel_timeline_create_from_engine(ce->engine, in pinned_timeline()
5356 struct intel_engine_cs *engine) in __execlists_context_alloc() argument
5365 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __execlists_context_alloc()
5370 if (INTEL_GEN(engine->i915) == 12) { in __execlists_context_alloc()
5375 ctx_obj = i915_gem_object_create_shmem(engine->i915, context_size); in __execlists_context_alloc()
5379 vma = i915_vma_instance(ctx_obj, &engine->gt->ggtt->vm, NULL); in __execlists_context_alloc()
5395 tl = intel_timeline_create(engine->gt); in __execlists_context_alloc()
5404 ring = intel_engine_create_ring(engine, (unsigned long)ce->ring); in __execlists_context_alloc()
5410 ret = populate_lr_context(ce, ctx_obj, engine, ring); in __execlists_context_alloc()
5412 drm_dbg(&engine->i915->drm, in __execlists_context_alloc()
5505 * When destroying the virtual engine, we have to be aware that in virtual_context_destroy()
5508 * due to preempt-to-busy). Before we can free the engine, we need in virtual_context_destroy()
5510 * accessing the engine. Flushing the tasklets requires process context, in virtual_context_destroy()
5511 * and since we can guard the resubmit onto the engine with an RCU read in virtual_context_destroy()
5512 * lock, we can delegate the free of the engine to an RCU worker. in virtual_context_destroy()
5530 * first engine we inspect being different each time. in virtual_engine_initial_hint()
5532 * NB This does not force us to execute on this engine, it will just in virtual_engine_initial_hint()
5551 /* Note: we must use a real engine class for setting up reg state */ in virtual_context_pin()
5603 /* Invalid selection, submit to a random engine in error */ in virtual_submission_mask()
5698 struct virtual_engine *ve = to_virtual_engine(rq->engine); in virtual_submit_request()
5752 struct virtual_engine *ve = to_virtual_engine(rq->engine); in virtual_bond_execute()
5756 allowed = ~to_request(signal)->engine->mask; in virtual_bond_execute()
5758 bond = virtual_find_bond(ve, to_request(signal)->engine); in virtual_bond_execute()
5801 * depends on the saturated state of the engine. We only compute in intel_execlists_create_virtual()
5804 * to this engine. Virtual engines encompass more than one physical in intel_execlists_create_virtual()
5805 * engine and so we cannot accurately tell in advance if one of those in intel_execlists_create_virtual()
5852 * The virtual engine implementation is tightly coupled to in intel_execlists_create_virtual()
5854 * into a tree inside each physical engine. We could support in intel_execlists_create_virtual()
5874 * on the physical engine). We use the engine class as a guide in intel_execlists_create_virtual()
5879 DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n", in intel_execlists_create_virtual()
5925 struct virtual_engine *de = to_virtual_engine(dst->engine); in intel_execlists_clone_virtual()
5941 int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, in intel_virtual_engine_attach_bond() argument
5945 struct virtual_engine *ve = to_virtual_engine(engine); in intel_virtual_engine_attach_bond()
5949 /* Sanity check the sibling is part of the virtual engine */ in intel_virtual_engine_attach_bond()
5978 intel_virtual_engine_get_sibling(struct intel_engine_cs *engine, in intel_virtual_engine_get_sibling() argument
5981 struct virtual_engine *ve = to_virtual_engine(engine); in intel_virtual_engine_get_sibling()
5989 void intel_execlists_show_requests(struct intel_engine_cs *engine, in intel_execlists_show_requests() argument
5996 const struct intel_engine_execlists *execlists = &engine->execlists; in intel_execlists_show_requests()
6002 spin_lock_irqsave(&engine->active.lock, flags); in intel_execlists_show_requests()
6006 list_for_each_entry(rq, &engine->active.requests, sched.link) { in intel_execlists_show_requests()
6054 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in intel_execlists_show_requests()
6073 spin_unlock_irqrestore(&engine->active.lock, flags); in intel_execlists_show_requests()
6076 void intel_lr_context_reset(struct intel_engine_cs *engine, in intel_lr_context_reset() argument
6092 restore_default_state(ce, engine); in intel_lr_context_reset()
6095 __execlists_update_reg_state(ce, engine, head); in intel_lr_context_reset()
6099 intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine) in intel_engine_in_execlists_submission_mode() argument
6101 return engine->set_default_submission == in intel_engine_in_execlists_submission_mode()