Lines Matching +full:never +full:- +full:post +full:- +full:merge +full:- +full:rules
2 * Copyright © 2008-2015 Intel Corporation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
25 #include <linux/dma-fence-array.h>
26 #include <linux/dma-fence-chain.h>
62 return dev_name(to_request(fence)->i915->drm.dev); in i915_fence_get_driver_name()
78 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) in i915_fence_get_timeline_name()
85 return ctx->name; in i915_fence_get_timeline_name()
116 GEM_BUG_ON(rq->guc_prio != GUC_PRIO_INIT && in i915_fence_release()
117 rq->guc_prio != GUC_PRIO_FINI); in i915_fence_release()
119 i915_request_free_capture_list(fetch_and_zero(&rq->capture_list)); in i915_fence_release()
120 if (rq->batch_res) { in i915_fence_release()
121 i915_vma_resource_put(rq->batch_res); in i915_fence_release()
122 rq->batch_res = NULL; in i915_fence_release()
132 i915_sw_fence_fini(&rq->submit); in i915_fence_release()
133 i915_sw_fence_fini(&rq->semaphore); in i915_fence_release()
139 * very careful in what rq->engine we poke. The virtual engine is in i915_fence_release()
140 * referenced via the rq->context and we released that ref during in i915_fence_release()
143 * the reserved engine->request_pool is the power management parking, in i915_fence_release()
144 * which must-not-fail, and that is only run on the physical engines. in i915_fence_release()
148 * not be unsubmitted again, so rq->engine and rq->execution_mask in i915_fence_release()
149 * at this point is stable. rq->execution_mask will be a single in i915_fence_release()
153 * power-of-two we assume that rq->engine may still be a virtual in i915_fence_release()
161 * execute on that engine and never returned to the virtual engine in i915_fence_release()
163 * know that if the rq->execution_mask is a single bit, rq->engine in i915_fence_release()
166 if (is_power_of_2(rq->execution_mask) && in i915_fence_release()
167 !cmpxchg(&rq->engine->request_pool, NULL, rq)) in i915_fence_release()
186 i915_sw_fence_complete(cb->fence); in irq_execute_cb()
195 if (llist_empty(&rq->execute_cb)) in __notify_execute_cb()
199 llist_del_all(&rq->execute_cb), in __notify_execute_cb()
201 fn(&cb->work); in __notify_execute_cb()
211 wrk->func(wrk); in irq_work_imm()
222 void *vaddr = rq->ring->vaddr; in __i915_request_fill()
225 head = rq->infix; in __i915_request_fill()
226 if (rq->postfix < head) { in __i915_request_fill()
227 memset(vaddr + head, val, rq->ring->size - head); in __i915_request_fill()
230 memset(vaddr + head, val, rq->postfix - head); in __i915_request_fill()
252 * is-banned?, or we know the request is already inflight. in i915_request_active_engine()
254 * Note that rq->engine is unstable, and so we double in i915_request_active_engine()
257 locked = READ_ONCE(rq->engine); in i915_request_active_engine()
258 spin_lock_irq(&locked->sched_engine->lock); in i915_request_active_engine()
259 while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { in i915_request_active_engine()
260 spin_unlock(&locked->sched_engine->lock); in i915_request_active_engine()
262 spin_lock(&locked->sched_engine->lock); in i915_request_active_engine()
271 spin_unlock_irq(&locked->sched_engine->lock); in i915_request_active_engine()
280 struct intel_gt *gt = rq->engine->gt; in __rq_watchdog_expired()
283 if (llist_add(&rq->watchdog.link, >->watchdog.list)) in __rq_watchdog_expired()
284 queue_work(gt->i915->unordered_wq, >->watchdog.work); in __rq_watchdog_expired()
294 struct i915_request_watchdog *wdg = &rq->watchdog; in __rq_init_watchdog()
296 hrtimer_setup(&wdg->timer, __rq_watchdog_expired, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in __rq_init_watchdog()
301 struct i915_request_watchdog *wdg = &rq->watchdog; in __rq_arm_watchdog()
302 struct intel_context *ce = rq->context; in __rq_arm_watchdog()
304 if (!ce->watchdog.timeout_us) in __rq_arm_watchdog()
309 hrtimer_start_range_ns(&wdg->timer, in __rq_arm_watchdog()
310 ns_to_ktime(ce->watchdog.timeout_us * in __rq_arm_watchdog()
318 struct i915_request_watchdog *wdg = &rq->watchdog; in __rq_cancel_watchdog()
320 if (hrtimer_try_to_cancel(&wdg->timer) > 0) in __rq_cancel_watchdog()
327 * i915_request_free_capture_list - Free a capture list
334 struct i915_capture_list *next = capture->next; in i915_request_free_capture_list()
336 i915_vma_resource_put(capture->vma_res); in i915_request_free_capture_list()
342 #define assert_capture_list_is_null(_rq) GEM_BUG_ON((_rq)->capture_list)
344 #define clear_capture_list(_rq) ((_rq)->capture_list = NULL)
363 GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit)); in i915_request_retire()
378 GEM_BUG_ON(!list_is_first(&rq->link, in i915_request_retire()
379 &i915_request_timeline(rq)->requests)); in i915_request_retire()
383 rq->ring->head = rq->postfix; in i915_request_retire()
386 spin_lock_irq(&rq->lock); in i915_request_retire()
387 dma_fence_signal_locked(&rq->fence); in i915_request_retire()
388 spin_unlock_irq(&rq->lock); in i915_request_retire()
391 if (test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) in i915_request_retire()
392 intel_rps_dec_waiters(&rq->engine->gt->rps); in i915_request_retire()
404 rq->engine->remove_active_request(rq); in i915_request_retire()
405 GEM_BUG_ON(!llist_empty(&rq->execute_cb)); in i915_request_retire()
407 __list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */ in i915_request_retire()
409 intel_context_exit(rq->context); in i915_request_retire()
410 intel_context_unpin(rq->context); in i915_request_retire()
412 i915_sched_node_fini(&rq->sched); in i915_request_retire()
427 tmp = list_first_entry(&tl->requests, typeof(*tmp), link); in i915_request_retire_upto()
435 return READ_ONCE(engine->execlists.active); in __engine_active()
448 * the GPU (preempt-to-busy). If that request is inside an in __request_in_flight()
474 * Note that the read of *execlists->active may race with the promotion in __request_in_flight()
475 * of execlists->pending[] to execlists->inflight[], overwriting in __request_in_flight()
476 * the value at *execlists->active. This is fine. The promotion implies in __request_in_flight()
478 * stuck -- if we do not see ourselves in *active, the inflight status in __request_in_flight()
482 if (!intel_context_inflight(signal->context)) in __request_in_flight()
486 for (port = __engine_active(signal->engine); in __request_in_flight()
489 if (rq->context == signal->context) { in __request_in_flight()
490 inflight = i915_seqno_passed(rq->fence.seqno, in __request_in_flight()
491 signal->fence.seqno); in __request_in_flight()
512 return -ENOMEM; in __await_execution()
514 cb->fence = &rq->submit; in __await_execution()
515 i915_sw_fence_await(cb->fence); in __await_execution()
516 init_irq_work(&cb->work, irq_execute_cb); in __await_execution()
531 if (llist_add(&cb->work.node.llist, &signal->execute_cb)) { in __await_execution()
544 case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */ in fatal_error()
545 case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */ in fatal_error()
554 GEM_BUG_ON(!fatal_error(rq->fence.error)); in __i915_request_skip()
556 if (rq->infix == rq->postfix) in __i915_request_skip()
559 RQ_TRACE(rq, "error: %d\n", rq->fence.error); in __i915_request_skip()
567 rq->infix = rq->postfix; in __i915_request_skip()
579 old = READ_ONCE(rq->fence.error); in i915_request_set_error_once()
583 } while (!try_cmpxchg(&rq->fence.error, &old, error)); in i915_request_set_error_once()
598 i915_request_set_error_once(rq, -EIO); in i915_request_mark_eio()
606 struct intel_engine_cs *engine = request->engine; in __i915_request_submit()
612 lockdep_assert_held(&engine->sched_engine->lock); in __i915_request_submit()
615 * With the advent of preempt-to-busy, we frequently encounter in __i915_request_submit()
624 * priority queue, under the sched_engine->lock. This ensures that the in __i915_request_submit()
626 * the request into the engine->active.list where it will be in __i915_request_submit()
628 * request, this would be a horrible use-after-free.) in __i915_request_submit()
631 list_del_init(&request->sched.link); in __i915_request_submit()
635 if (unlikely(!intel_context_is_schedulable(request->context))) in __i915_request_submit()
636 i915_request_set_error_once(request, -EIO); in __i915_request_submit()
638 if (unlikely(fatal_error(request->fence.error))) in __i915_request_submit()
648 * tries to access memory across the bus (perf stat -e bus-cycles). in __i915_request_submit()
657 if (request->sched.semaphores && in __i915_request_submit()
658 i915_sw_fence_signaled(&request->semaphore)) in __i915_request_submit()
659 engine->saturated |= request->sched.semaphores; in __i915_request_submit()
661 engine->emit_fini_breadcrumb(request, in __i915_request_submit()
662 request->ring->vaddr + request->postfix); in __i915_request_submit()
665 if (engine->bump_serial) in __i915_request_submit()
666 engine->bump_serial(engine); in __i915_request_submit()
668 engine->serial++; in __i915_request_submit()
672 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); in __i915_request_submit()
673 engine->add_active_request(request); in __i915_request_submit()
675 clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags); in __i915_request_submit()
676 set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); in __i915_request_submit()
679 * XXX Rollback bonded-execution on __i915_request_unsubmit()? in __i915_request_submit()
681 * In the future, perhaps when we have an active time-slicing scheduler, in __i915_request_submit()
685 * preempt-to-idle cycle on the target engine, all the while the in __i915_request_submit()
691 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) in __i915_request_submit()
699 struct intel_engine_cs *engine = request->engine; in i915_request_submit()
702 /* Will be called from irq-context when using foreign fences. */ in i915_request_submit()
703 spin_lock_irqsave(&engine->sched_engine->lock, flags); in i915_request_submit()
707 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); in i915_request_submit()
712 struct intel_engine_cs *engine = request->engine; in __i915_request_unsubmit()
715 * Only unwind in reverse order, required so that the per-context list in __i915_request_unsubmit()
721 lockdep_assert_held(&engine->sched_engine->lock); in __i915_request_unsubmit()
730 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); in __i915_request_unsubmit()
731 clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); in __i915_request_unsubmit()
732 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) in __i915_request_unsubmit()
736 if (request->sched.semaphores && __i915_request_has_started(request)) in __i915_request_unsubmit()
737 request->sched.semaphores = 0; in __i915_request_unsubmit()
740 * We don't need to wake_up any waiters on request->execute, they in __i915_request_unsubmit()
741 * will get woken by any other event or us re-adding this request in __i915_request_unsubmit()
750 struct intel_engine_cs *engine = request->engine; in i915_request_unsubmit()
753 /* Will be called from irq-context when using foreign fences. */ in i915_request_unsubmit()
754 spin_lock_irqsave(&engine->sched_engine->lock, flags); in i915_request_unsubmit()
758 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); in i915_request_unsubmit()
766 set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags); in i915_request_cancel()
768 intel_context_cancel_request(rq->context, rq); in i915_request_cancel()
781 if (unlikely(fence->error)) in submit_notify()
782 i915_request_set_error_once(request, fence->error); in submit_notify()
795 request->engine->submit_request(request); in submit_notify()
828 list_for_each_entry_safe(rq, rn, &tl->requests, link) in retire_requests()
849 if (list_empty(&tl->requests)) in request_alloc_slow()
852 /* Move our oldest request to the slab-cache (if not in use!) */ in request_alloc_slow()
853 rq = list_first_entry(&tl->requests, typeof(*rq), link); in request_alloc_slow()
862 rq = list_last_entry(&tl->requests, typeof(*rq), link); in request_alloc_slow()
863 cond_synchronize_rcu(rq->rcustate); in request_alloc_slow()
876 spin_lock_init(&rq->lock); in __i915_request_ctor()
877 i915_sched_node_init(&rq->sched); in __i915_request_ctor()
878 i915_sw_fence_init(&rq->submit, submit_notify); in __i915_request_ctor()
879 i915_sw_fence_init(&rq->semaphore, semaphore_notify); in __i915_request_ctor()
882 rq->batch_res = NULL; in __i915_request_ctor()
884 init_llist_head(&rq->execute_cb); in __i915_request_ctor()
888 #define clear_batch_ptr(_rq) ((_rq)->batch = NULL)
896 struct intel_timeline *tl = ce->timeline; in __i915_request_create()
914 * the RCU lookup, we change chase the request->engine pointer, in __i915_request_create()
915 * read the request->global_seqno and increment the reference count. in __i915_request_create()
924 * Before we increment the refcount, we chase the request->engine in __i915_request_create()
931 * active request - which it won't be and restart the lookup. in __i915_request_create()
938 rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp); in __i915_request_create()
940 ret = -ENOMEM; in __i915_request_create()
945 rq->context = ce; in __i915_request_create()
946 rq->engine = ce->engine; in __i915_request_create()
947 rq->ring = ce->ring; in __i915_request_create()
948 rq->execution_mask = ce->engine->mask; in __i915_request_create()
949 rq->i915 = ce->engine->i915; in __i915_request_create()
955 dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock, in __i915_request_create()
956 tl->fence_context, seqno); in __i915_request_create()
958 RCU_INIT_POINTER(rq->timeline, tl); in __i915_request_create()
959 rq->hwsp_seqno = tl->hwsp_seqno; in __i915_request_create()
962 rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */ in __i915_request_create()
964 rq->guc_prio = GUC_PRIO_INIT; in __i915_request_create()
967 i915_sw_fence_reinit(&i915_request_get(rq)->submit); in __i915_request_create()
968 i915_sw_fence_reinit(&i915_request_get(rq)->semaphore); in __i915_request_create()
970 i915_sched_node_reinit(&rq->sched); in __i915_request_create()
976 GEM_BUG_ON(!llist_empty(&rq->execute_cb)); in __i915_request_create()
977 GEM_BUG_ON(rq->batch_res); in __i915_request_create()
991 rq->reserved_space = in __i915_request_create()
992 2 * rq->engine->emit_fini_breadcrumb_dw * sizeof(u32); in __i915_request_create()
996 * should we detect the updated seqno part-way through the in __i915_request_create()
997 * GPU processing the request, we never over-estimate the in __i915_request_create()
1000 rq->head = rq->ring->emit; in __i915_request_create()
1002 ret = rq->engine->request_alloc(rq); in __i915_request_create()
1006 rq->infix = rq->ring->emit; /* end of header; start of user payload */ in __i915_request_create()
1009 list_add_tail_rcu(&rq->link, &tl->requests); in __i915_request_create()
1014 ce->ring->emit = rq->head; in __i915_request_create()
1017 GEM_BUG_ON(!list_empty(&rq->sched.signalers_list)); in __i915_request_create()
1018 GEM_BUG_ON(!list_empty(&rq->sched.waiters_list)); in __i915_request_create()
1037 /* Move our oldest request to the slab-cache (if not in use!) */ in i915_request_create()
1038 rq = list_first_entry(&tl->requests, typeof(*rq), link); in i915_request_create()
1039 if (!list_is_last(&rq->link, &tl->requests)) in i915_request_create()
1049 rq->cookie = lockdep_pin_lock(&tl->mutex); in i915_request_create()
1064 if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline)) in i915_request_await_start()
1081 struct list_head *pos = READ_ONCE(signal->link.prev); in i915_request_await_start()
1089 if (pos == &rcu_dereference(signal->timeline)->requests) in i915_request_await_start()
1103 if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) { in i915_request_await_start()
1108 fence = &prev->fence; in i915_request_await_start()
1116 err = i915_sw_fence_await_dma_fence(&rq->submit, in i915_request_await_start()
1137 * See the are-we-too-late? check in __i915_request_submit(). in already_busywaiting()
1139 return rq->sched.semaphores | READ_ONCE(rq->engine->saturated); in already_busywaiting()
1147 const int has_token = GRAPHICS_VER(to->engine->i915) >= 12; in __emit_semaphore_wait()
1152 GEM_BUG_ON(GRAPHICS_VER(to->engine->i915) < 8); in __emit_semaphore_wait()
1169 * Using greater-than-or-equal here means we have to worry in __emit_semaphore_wait()
1172 * for the old (pre-wrap) values do not see the much smaller in __emit_semaphore_wait()
1173 * (post-wrap) values than they were expecting (and so wait in __emit_semaphore_wait()
1196 return to->engine->gt->ggtt == from->engine->gt->ggtt; in can_use_semaphore_wait()
1204 const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask; in emit_semaphore_wait()
1205 struct i915_sw_fence *wait = &to->submit; in emit_semaphore_wait()
1210 if (!intel_context_use_semaphores(to->context)) in emit_semaphore_wait()
1220 * lose the fence->error propagation. in emit_semaphore_wait()
1222 if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN) in emit_semaphore_wait()
1236 if (__emit_semaphore_wait(to, from, from->fence.seqno)) in emit_semaphore_wait()
1239 to->sched.semaphores |= mask; in emit_semaphore_wait()
1240 wait = &to->semaphore; in emit_semaphore_wait()
1244 &from->fence, 0, in emit_semaphore_wait()
1252 fence->context, in intel_timeline_sync_has_start()
1253 fence->seqno - 1); in intel_timeline_sync_has_start()
1259 return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1); in intel_timeline_sync_set_start()
1268 GEM_BUG_ON(intel_context_is_barrier(from->context)); in __i915_request_await_execution()
1277 &from->fence)) in __i915_request_await_execution()
1308 * (with just an ordinary intra-engine fence dependency between the in __i915_request_await_execution()
1314 intel_engine_has_semaphores(to->engine) && in __i915_request_await_execution()
1316 err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); in __i915_request_await_execution()
1321 /* Couple the dependency tree for PI on this exposed to->fence */ in __i915_request_await_execution()
1322 if (to->engine->sched_engine->schedule) { in __i915_request_await_execution()
1323 err = i915_sched_node_add_dependency(&to->sched, in __i915_request_await_execution()
1324 &from->sched, in __i915_request_await_execution()
1331 &from->fence); in __i915_request_await_execution()
1344 rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN; in mark_external()
1351 return i915_sw_fence_await_dma_fence(&rq->submit, fence, in __i915_request_await_external()
1352 i915_fence_context_timeout(rq->i915, in __i915_request_await_external()
1353 fence->context), in __i915_request_await_external()
1369 if (!dma_fence_is_i915(chain->fence)) { in i915_request_await_external()
1374 err = i915_request_await_dma_fence(rq, chain->fence); in i915_request_await_external()
1385 return intel_context_is_parallel(rq->context); in is_parallel_rq()
1390 return intel_context_to_parent(rq->context); in request_to_parent()
1413 /* XXX Error for signal-on-any fence arrays */ in i915_request_await_execution()
1415 child = array->fences; in i915_request_await_execution()
1416 nchild = array->num_fences; in i915_request_await_execution()
1422 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) in i915_request_await_execution()
1425 if (fence->context == rq->fence.context) in i915_request_await_execution()
1443 } while (--nchild); in i915_request_await_execution()
1459 if (to->engine == READ_ONCE(from->engine)) in await_request_submit()
1460 return i915_sw_fence_await_sw_fence_gfp(&to->submit, in await_request_submit()
1461 &from->submit, in await_request_submit()
1473 GEM_BUG_ON(to->timeline == from->timeline); in i915_request_await_request()
1476 i915_sw_fence_set_error_once(&to->submit, from->fence.error); in i915_request_await_request()
1480 if (to->engine->sched_engine->schedule) { in i915_request_await_request()
1481 ret = i915_sched_node_add_dependency(&to->sched, in i915_request_await_request()
1482 &from->sched, in i915_request_await_request()
1488 if (!intel_engine_uses_guc(to->engine) && in i915_request_await_request()
1489 is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask))) in i915_request_await_request()
1507 * Note that if the fence-array was created in signal-on-any mode, in i915_request_await_dma_fence()
1509 * we don't currently store which mode the fence-array is operating in i915_request_await_dma_fence()
1510 * in. Fortunately, the only user of signal-on-any is private to in i915_request_await_dma_fence()
1511 * amdgpu and we should not see any incoming fence-array from in i915_request_await_dma_fence()
1512 * sync-file being in signal-on-any mode. in i915_request_await_dma_fence()
1517 child = array->fences; in i915_request_await_dma_fence()
1518 nchild = array->num_fences; in i915_request_await_dma_fence()
1524 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) in i915_request_await_dma_fence()
1530 * that requests are submitted in-order through each ring. in i915_request_await_dma_fence()
1532 if (fence->context == rq->fence.context) in i915_request_await_dma_fence()
1536 if (fence->context && in i915_request_await_dma_fence()
1552 if (fence->context) in i915_request_await_dma_fence()
1555 } while (--nchild); in i915_request_await_dma_fence()
1561 * i915_request_await_deps - set this request to (async) wait upon a struct
1572 for (i = 0; i < deps->num_deps; ++i) { in i915_request_await_deps()
1573 err = i915_request_await_dma_fence(rq, deps->fences[i]); in i915_request_await_deps()
1582 * i915_request_await_object - set this request to (async) wait upon a bo
1592 * - If there is an outstanding write request to the object, the new
1596 * - If we are a write request (pending_write_domain is set), the new
1610 dma_resv_for_each_fence(&cursor, obj->base.resv, in i915_request_await_object()
1622 struct intel_huc *huc = &rq->context->engine->gt->uc.huc; in i915_request_await_huc()
1625 if (!rcu_access_pointer(rq->context->gem_context)) in i915_request_await_huc()
1629 i915_sw_fence_await_sw_fence(&rq->submit, in i915_request_await_huc()
1630 &huc->delayed_load.fence, in i915_request_await_huc()
1631 &rq->hucq); in i915_request_await_huc()
1642 prev = request_to_parent(rq)->parallel.last_rq; in __i915_request_ensure_parallel_ordering()
1645 i915_sw_fence_await_sw_fence(&rq->submit, in __i915_request_ensure_parallel_ordering()
1646 &prev->submit, in __i915_request_ensure_parallel_ordering()
1647 &rq->submitq); in __i915_request_ensure_parallel_ordering()
1649 if (rq->engine->sched_engine->schedule) in __i915_request_ensure_parallel_ordering()
1650 __i915_sched_node_add_dependency(&rq->sched, in __i915_request_ensure_parallel_ordering()
1651 &prev->sched, in __i915_request_ensure_parallel_ordering()
1652 &rq->dep, in __i915_request_ensure_parallel_ordering()
1658 request_to_parent(rq)->parallel.last_rq = i915_request_get(rq); in __i915_request_ensure_parallel_ordering()
1665 return to_request(__i915_active_fence_set(&timeline->last_request, in __i915_request_ensure_parallel_ordering()
1666 &rq->fence)); in __i915_request_ensure_parallel_ordering()
1677 prev = to_request(__i915_active_fence_set(&timeline->last_request, in __i915_request_ensure_ordering()
1678 &rq->fence)); in __i915_request_ensure_ordering()
1681 bool uses_guc = intel_engine_uses_guc(rq->engine); in __i915_request_ensure_ordering()
1682 bool pow2 = is_power_of_2(READ_ONCE(prev->engine)->mask | in __i915_request_ensure_ordering()
1683 rq->engine->mask); in __i915_request_ensure_ordering()
1684 bool same_context = prev->context == rq->context; in __i915_request_ensure_ordering()
1688 * we need to be wary in case the timeline->last_request in __i915_request_ensure_ordering()
1693 i915_seqno_passed(prev->fence.seqno, in __i915_request_ensure_ordering()
1694 rq->fence.seqno)); in __i915_request_ensure_ordering()
1697 i915_sw_fence_await_sw_fence(&rq->submit, in __i915_request_ensure_ordering()
1698 &prev->submit, in __i915_request_ensure_ordering()
1699 &rq->submitq); in __i915_request_ensure_ordering()
1701 __i915_sw_fence_await_dma_fence(&rq->submit, in __i915_request_ensure_ordering()
1702 &prev->fence, in __i915_request_ensure_ordering()
1703 &rq->dmaq); in __i915_request_ensure_ordering()
1704 if (rq->engine->sched_engine->schedule) in __i915_request_ensure_ordering()
1705 __i915_sched_node_add_dependency(&rq->sched, in __i915_request_ensure_ordering()
1706 &prev->sched, in __i915_request_ensure_ordering()
1707 &rq->dep, in __i915_request_ensure_ordering()
1731 if (rq->engine->class == VIDEO_DECODE_CLASS) in __i915_request_add_to_timeline()
1741 * we embed the hooks into our request struct -- at the cost of in __i915_request_add_to_timeline()
1742 * having to have specialised no-allocation interfaces (which will in __i915_request_add_to_timeline()
1745 * A second benefit to open-coding i915_request_await_request is in __i915_request_add_to_timeline()
1746 * that we can apply a slight variant of the rules specialised in __i915_request_add_to_timeline()
1748 * If we consider the case of virtual engine, we must emit a dma-fence in __i915_request_add_to_timeline()
1756 * rules for parallel requests are that they must be submitted in the in __i915_request_add_to_timeline()
1772 * Make sure that no request gazumped us - if it was allocated after in __i915_request_add_to_timeline()
1776 GEM_BUG_ON(timeline->seqno != rq->fence.seqno); in __i915_request_add_to_timeline()
1788 struct intel_engine_cs *engine = rq->engine; in __i915_request_commit()
1789 struct intel_ring *ring = rq->ring; in __i915_request_commit()
1799 GEM_BUG_ON(rq->reserved_space > ring->space); in __i915_request_commit()
1800 rq->reserved_space = 0; in __i915_request_commit()
1801 rq->emitted_jiffies = jiffies; in __i915_request_commit()
1805 * should we detect the updated seqno part-way through the in __i915_request_commit()
1806 * GPU processing the request, we never over-estimate the in __i915_request_commit()
1809 cs = intel_ring_begin(rq, engine->emit_fini_breadcrumb_dw); in __i915_request_commit()
1811 rq->postfix = intel_ring_offset(rq, cs); in __i915_request_commit()
1818 i915_sw_fence_commit(&rq->semaphore); in __i915_request_queue_bh()
1819 i915_sw_fence_commit(&rq->submit); in __i915_request_queue_bh()
1828 * request - i.e. we may want to preempt the current request in order in __i915_request_queue()
1836 if (attr && rq->engine->sched_engine->schedule) in __i915_request_queue()
1837 rq->engine->sched_engine->schedule(rq, attr); in __i915_request_queue()
1850 lockdep_assert_held(&tl->mutex); in i915_request_add()
1851 lockdep_unpin_lock(&tl->mutex, rq->cookie); in i915_request_add()
1858 ctx = rcu_dereference(rq->context->gem_context); in i915_request_add()
1860 attr = ctx->sched; in i915_request_add()
1865 mutex_unlock(&tl->mutex); in i915_request_add()
1931 timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns); in __i915_spin_request()
1934 if (dma_fence_is_signaled(&rq->fence)) in __i915_spin_request()
1958 wake_up_process(fetch_and_zero(&wait->tsk)); in request_wait_wake()
1962 * i915_request_wait_timeout - wait until execution of request has finished
1975 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1978 * NOTE: This function has the same wait semantics as dma-fence.
1991 if (dma_fence_is_signaled(&rq->fence)) in i915_request_wait_timeout()
1995 return -ETIME; in i915_request_wait_timeout()
2000 * We must never wait on the GPU while holding a lock as we in i915_request_wait_timeout()
2005 mutex_acquire(&rq->engine->gt->reset.mutex.dep_map, 0, 0, _THIS_IP_); in i915_request_wait_timeout()
2024 * The scheme used for low-latency IO is called "hybrid interrupt in i915_request_wait_timeout()
2050 if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake)) in i915_request_wait_timeout()
2069 __intel_engine_flush_submission(rq->engine, false); in i915_request_wait_timeout()
2074 if (dma_fence_is_signaled(&rq->fence)) in i915_request_wait_timeout()
2078 timeout = -ERESTARTSYS; in i915_request_wait_timeout()
2083 timeout = -ETIME; in i915_request_wait_timeout()
2092 dma_fence_remove_callback(&rq->fence, &wait.cb); in i915_request_wait_timeout()
2096 mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_); in i915_request_wait_timeout()
2102 * i915_request_wait - wait until execution of request has finished
2112 * be zero or -ETIME if the request is unfinished after the timeout expires.
2113 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
2116 * NOTE: This function behaves differently from dma-fence wait semantics for
2117 * timeout = 0. It returns 0 on success, and -ETIME if not signaled.
2126 return -ETIME; in i915_request_wait()
2137 if (attr->priority == I915_PRIORITY_INVALID) in print_sched_attr()
2140 x += snprintf(buf + x, len - x, in print_sched_attr()
2141 " prio=%d", attr->priority); in print_sched_attr()
2152 return intel_engine_is_virtual(rq->engine) ? 'V' : 'R'; in queue_status()
2165 if (!i915_sw_fence_signaled(&rq->semaphore)) in run_status()
2173 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) in fence_status()
2176 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) in fence_status()
2177 return "-"; in fence_status()
2187 const char *name = rq->fence.ops->get_timeline_name((struct dma_fence *)&rq->fence); in i915_request_show()
2196 * - initial status upon being submitted by the user in i915_request_show()
2198 * - the request is not ready for execution as it is waiting in i915_request_show()
2202 * - all fences the request was waiting on have been signaled, in i915_request_show()
2206 * - a ready request may still need to wait on semaphores in i915_request_show()
2210 * - same as ready, but queued over multiple backends in i915_request_show()
2213 * - the request has been transferred from the backend queue and in i915_request_show()
2216 * - a completed request may still be regarded as executing, its in i915_request_show()
2221 x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf)); in i915_request_show()
2226 rq->fence.context, rq->fence.seqno, in i915_request_show()
2230 jiffies_to_msecs(jiffies - rq->emitted_jiffies), in i915_request_show()
2238 return ring == i915_ggtt_offset(rq->ring->vma); in engine_match_ring()
2247 if (!intel_engine_is_virtual(rq->engine)) in match_ring()
2248 return engine_match_ring(rq->engine, rq); in match_ring()
2252 while ((engine = intel_engine_get_sibling(rq->engine, i++))) { in match_ring()
2297 return -ENOMEM; in i915_request_module_init()
2310 return -ENOMEM; in i915_request_module_init()