Lines Matching +full:cs +full:- +full:out
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
73 request = mock_request(rcs0(i915)->kernel_context, HZ / 10); in igt_add_request()
75 return -ENOMEM; in igt_add_request()
87 int err = -EINVAL; in igt_wait_request()
91 request = mock_request(rcs0(i915)->kernel_context, T); in igt_wait_request()
93 return -ENOMEM; in igt_wait_request()
97 if (i915_request_wait(request, 0, 0) != -ETIME) { in igt_wait_request()
102 if (i915_request_wait(request, 0, T) != -ETIME) { in igt_wait_request()
114 if (i915_request_wait(request, 0, 0) != -ETIME) { in igt_wait_request()
124 if (i915_request_wait(request, 0, T / 2) != -ETIME) { in igt_wait_request()
129 if (i915_request_wait(request, 0, T) == -ETIME) { in igt_wait_request()
130 pr_err("request wait timed out!\n"); in igt_wait_request()
139 if (i915_request_wait(request, 0, T) == -ETIME) { in igt_wait_request()
140 pr_err("request wait timed out when already complete!\n"); in igt_wait_request()
156 int err = -EINVAL; in igt_fence_wait()
160 request = mock_request(rcs0(i915)->kernel_context, T); in igt_fence_wait()
162 return -ENOMEM; in igt_fence_wait()
164 if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) { in igt_fence_wait()
166 goto out; in igt_fence_wait()
171 if (dma_fence_is_signaled(&request->fence)) { in igt_fence_wait()
173 goto out; in igt_fence_wait()
176 if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) { in igt_fence_wait()
178 goto out; in igt_fence_wait()
181 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { in igt_fence_wait()
182 pr_err("fence wait timed out (expected success)!\n"); in igt_fence_wait()
183 goto out; in igt_fence_wait()
186 if (!dma_fence_is_signaled(&request->fence)) { in igt_fence_wait()
188 goto out; in igt_fence_wait()
191 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) { in igt_fence_wait()
192 pr_err("fence wait timed out when complete (expected success)!\n"); in igt_fence_wait()
193 goto out; in igt_fence_wait()
197 out: in igt_fence_wait()
208 int err = -EINVAL; in igt_request_rewind()
217 err = -ENOMEM; in igt_request_rewind()
231 err = -ENOMEM; in igt_request_rewind()
244 request->engine->submit_request(request); in igt_request_rewind()
248 if (i915_request_wait(vip, 0, HZ) == -ETIME) { in igt_request_rewind()
249 pr_err("timed out waiting for high priority request\n"); in igt_request_rewind()
293 const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1; in __igt_breadcrumbs_smoketest()
294 const unsigned int total = 4 * t->ncontexts + 1; in __igt_breadcrumbs_smoketest()
312 return -ENOMEM; in __igt_breadcrumbs_smoketest()
316 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
326 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
334 err = -ENOMEM; in __igt_breadcrumbs_smoketest()
343 t->contexts[order[n] % t->ncontexts]; in __igt_breadcrumbs_smoketest()
347 ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx); in __igt_breadcrumbs_smoketest()
349 rq = t->request_alloc(ce); in __igt_breadcrumbs_smoketest()
357 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, in __igt_breadcrumbs_smoketest()
366 &rq->fence, in __igt_breadcrumbs_smoketest()
380 if (!wait_event_timeout(wait->wait, in __igt_breadcrumbs_smoketest()
383 struct i915_request *rq = requests[count - 1]; in __igt_breadcrumbs_smoketest()
385 pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n", in __igt_breadcrumbs_smoketest()
386 atomic_read(&wait->pending), count, in __igt_breadcrumbs_smoketest()
387 rq->fence.context, rq->fence.seqno, in __igt_breadcrumbs_smoketest()
388 t->engine->name); in __igt_breadcrumbs_smoketest()
391 intel_gt_set_wedged(t->engine->gt); in __igt_breadcrumbs_smoketest()
394 err = -EIO; in __igt_breadcrumbs_smoketest()
401 &rq->fence.flags)) { in __igt_breadcrumbs_smoketest()
403 rq->fence.context, rq->fence.seqno); in __igt_breadcrumbs_smoketest()
404 err = -EINVAL; in __igt_breadcrumbs_smoketest()
422 atomic_long_add(num_fences, &t->num_fences); in __igt_breadcrumbs_smoketest()
423 atomic_long_add(num_waits, &t->num_waits); in __igt_breadcrumbs_smoketest()
453 return -ENOMEM; in mock_breadcrumbs_smoketest()
457 ret = -ENOMEM; in mock_breadcrumbs_smoketest()
462 t.contexts[n] = mock_context(t.engine->i915, "mock"); in mock_breadcrumbs_smoketest()
464 ret = -ENOMEM; in mock_breadcrumbs_smoketest()
525 return -ENOMEM; in i915_request_mock_selftests()
527 with_intel_runtime_pm(&i915->runtime_pm, wakeref) in i915_request_mock_selftests()
540 int err = -ENODEV; in live_nop_request()
553 err = igt_live_test_begin(&t, i915, __func__, engine->name); in live_nop_request()
565 request = i915_request_create(engine->kernel_context); in live_nop_request()
575 * and submission of our breadcrumbs - in live_nop_request()
603 engine->name, in live_nop_request()
633 intel_gt_chipset_flush(&i915->gt); in empty_batch()
635 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); in empty_batch()
666 request = i915_request_create(engine->kernel_context); in empty_request()
670 err = engine->emit_bb_start(request, in empty_request()
671 batch->node.start, in empty_request()
672 batch->node.size, in empty_request()
707 err = igt_live_test_begin(&t, i915, __func__, engine->name); in live_empty_request()
751 engine->name, in live_empty_request()
774 vma = i915_vma_instance(obj, i915->gt.vm, NULL); in recursive_batch()
792 *cmd++ = lower_32_bits(vma->node.start); in recursive_batch()
793 *cmd++ = upper_32_bits(vma->node.start); in recursive_batch()
796 *cmd++ = lower_32_bits(vma->node.start); in recursive_batch()
799 *cmd++ = lower_32_bits(vma->node.start); in recursive_batch()
806 intel_gt_chipset_flush(&i915->gt); in recursive_batch()
819 cmd = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); in recursive_batch_resolve()
825 __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd)); in recursive_batch_resolve()
826 i915_gem_object_unpin_map(batch->obj); in recursive_batch_resolve()
828 intel_gt_chipset_flush(batch->vm->gt); in recursive_batch_resolve()
846 * send a recursive batch to each engine - checking that we don't in live_all_engines()
852 return -ENOMEM; in live_all_engines()
877 err = i915_request_await_object(request[idx], batch->obj, 0); in live_all_engines()
882 err = engine->emit_bb_start(request[idx], in live_all_engines()
883 batch->node.start, in live_all_engines()
884 batch->node.size, in live_all_engines()
887 request[idx]->batch = batch; in live_all_engines()
900 __func__, engine->name); in live_all_engines()
901 err = -EINVAL; in live_all_engines()
922 __func__, engine->name, err); in live_all_engines()
962 * tests that we don't execute requests out of order, even though in live_sequential_engines()
968 return -ENOMEM; in live_sequential_engines()
982 __func__, engine->name, err); in live_sequential_engines()
991 __func__, engine->name, err); in live_sequential_engines()
997 &prev->fence); in live_sequential_engines()
1001 __func__, engine->name, err); in live_sequential_engines()
1007 batch->obj, false); in live_sequential_engines()
1012 err = engine->emit_bb_start(request[idx], in live_sequential_engines()
1013 batch->node.start, in live_sequential_engines()
1014 batch->node.size, in live_sequential_engines()
1017 request[idx]->batch = batch; in live_sequential_engines()
1037 __func__, engine->name); in live_sequential_engines()
1038 err = -EINVAL; in live_sequential_engines()
1042 err = recursive_batch_resolve(request[idx]->batch); in live_sequential_engines()
1054 __func__, engine->name, err); in live_sequential_engines()
1072 cmd = i915_gem_object_pin_map(request[idx]->batch->obj, in live_sequential_engines()
1077 __i915_gem_object_flush_map(request[idx]->batch->obj, in live_sequential_engines()
1079 i915_gem_object_unpin_map(request[idx]->batch->obj); in live_sequential_engines()
1081 intel_gt_chipset_flush(engine->gt); in live_sequential_engines()
1084 i915_vma_put(request[idx]->batch); in live_sequential_engines()
1105 rq = i915_request_create(engine->kernel_context); in __live_parallel_engine1()
1116 err = -ETIME; in __live_parallel_engine1()
1125 pr_info("%s: %lu request + sync\n", engine->name, count); in __live_parallel_engine1()
1141 rq = i915_request_create(engine->kernel_context); in __live_parallel_engineN()
1152 pr_info("%s: %lu requests\n", engine->name, count); in __live_parallel_engineN()
1158 if (atomic_dec_and_test(&i915->selftest.counter)) { in wake_all()
1159 wake_up_var(&i915->selftest.counter); in wake_all()
1171 if (wait_var_event_timeout(&i915->selftest.counter, in wait_for_all()
1172 !atomic_read(&i915->selftest.counter), in wait_for_all()
1176 return -ETIME; in wait_for_all()
1192 if (igt_spinner_init(&spin, engine->gt)) { in __live_parallel_spin()
1193 wake_all(engine->i915); in __live_parallel_spin()
1194 return -ENOMEM; in __live_parallel_spin()
1199 engine->kernel_context, in __live_parallel_spin()
1204 if (err == -ENODEV) in __live_parallel_spin()
1206 wake_all(engine->i915); in __live_parallel_spin()
1214 err = wait_for_all(engine->i915); in __live_parallel_spin()
1216 pr_err("Failed to start spinner on %s\n", engine->name); in __live_parallel_spin()
1217 err = -EINVAL; in __live_parallel_spin()
1222 err = -EIO; in __live_parallel_spin()
1252 return -ENOMEM; in live_parallel_engines()
1264 atomic_set(&i915->selftest.counter, nengines); in live_parallel_engines()
1270 engine->name); in live_parallel_engines()
1295 err = -EIO; in live_parallel_engines()
1317 if (HAS_EXECLISTS(ctx->i915)) in max_batches()
1326 ret = rq->ring->size - rq->reserved_space; in max_batches()
1329 sz = rq->ring->emit - rq->head; in max_batches()
1331 sz += rq->ring->size; in max_batches()
1362 wakeref = intel_runtime_pm_get(&i915->runtime_pm); in live_breadcrumbs_smoketest()
1372 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1378 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1388 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1395 ret = -ENOMEM; in live_breadcrumbs_smoketest()
1417 smoke[idx].max_batch, engine->name); in live_breadcrumbs_smoketest()
1475 intel_runtime_pm_put(&i915->runtime_pm, wakeref); in live_breadcrumbs_smoketest()
1491 if (intel_gt_is_wedged(&i915->gt)) in i915_request_live_selftests()
1502 rq = intel_engine_create_kernel_request(ce->engine); in switch_to_kernel_sync()
1506 fence = i915_active_fence_get(&ce->timeline->last_request); in switch_to_kernel_sync()
1515 err = -ETIME; in switch_to_kernel_sync()
1518 while (!err && !intel_engine_is_idle(ce->engine)) in switch_to_kernel_sync()
1519 intel_engine_flush_submission(ce->engine); in switch_to_kernel_sync()
1542 return *a - *b; in cmp_u32()
1563 u64 ns = i915_cs_timestamp_ticks_to_ns(engine->i915, cycles); in cycles_to_ns()
1568 static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset) in emit_timestamp_store() argument
1570 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in emit_timestamp_store()
1571 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base))); in emit_timestamp_store()
1572 *cs++ = offset; in emit_timestamp_store()
1573 *cs++ = 0; in emit_timestamp_store()
1575 return cs; in emit_timestamp_store()
1578 static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value) in emit_store_dw() argument
1580 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_store_dw()
1581 *cs++ = offset; in emit_store_dw()
1582 *cs++ = 0; in emit_store_dw()
1583 *cs++ = value; in emit_store_dw()
1585 return cs; in emit_store_dw()
1588 static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset) in emit_semaphore_poll() argument
1590 *cs++ = MI_SEMAPHORE_WAIT | in emit_semaphore_poll()
1594 *cs++ = value; in emit_semaphore_poll()
1595 *cs++ = offset; in emit_semaphore_poll()
1596 *cs++ = 0; in emit_semaphore_poll()
1598 return cs; in emit_semaphore_poll()
1601 static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value) in emit_semaphore_poll_until() argument
1603 return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset); in emit_semaphore_poll_until()
1614 return memset32(ce->engine->status_page.addr + 1000, 0, 21); in hwsp_scratch()
1619 return (i915_ggtt_offset(ce->engine->status_page.vma) + in hwsp_offset()
1629 u32 *cs; in measure_semaphore_response() local
1641 * Semaphore latency: B - A in measure_semaphore_response()
1644 semaphore_set(sema, -1); in measure_semaphore_response()
1650 cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed)); in measure_semaphore_response()
1651 if (IS_ERR(cs)) { in measure_semaphore_response()
1653 err = PTR_ERR(cs); in measure_semaphore_response()
1657 cs = emit_store_dw(cs, offset, 0); in measure_semaphore_response()
1659 cs = emit_semaphore_poll_until(cs, offset, i); in measure_semaphore_response()
1660 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_semaphore_response()
1661 cs = emit_store_dw(cs, offset, 0); in measure_semaphore_response()
1664 intel_ring_advance(rq, cs); in measure_semaphore_response()
1668 err = -EIO; in measure_semaphore_response()
1674 cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_semaphore_response()
1679 err = -EIO; in measure_semaphore_response()
1683 elapsed[i - 1] = sema[i] - cycles; in measure_semaphore_response()
1688 ce->engine->name, cycles >> TF_BIAS, in measure_semaphore_response()
1689 cycles_to_ns(ce->engine, cycles)); in measure_semaphore_response()
1691 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_semaphore_response()
1694 intel_gt_set_wedged(ce->engine->gt); in measure_semaphore_response()
1703 u32 *cs; in measure_idle_dispatch() local
1715 * Submission latency: B - A in measure_idle_dispatch()
1721 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_idle_dispatch()
1731 cs = intel_ring_begin(rq, 4); in measure_idle_dispatch()
1732 if (IS_ERR(cs)) { in measure_idle_dispatch()
1734 err = PTR_ERR(cs); in measure_idle_dispatch()
1738 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_idle_dispatch()
1740 intel_ring_advance(rq, cs); in measure_idle_dispatch()
1744 elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_idle_dispatch()
1750 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_idle_dispatch()
1755 elapsed[i] = sema[i] - elapsed[i]; in measure_idle_dispatch()
1759 ce->engine->name, cycles >> TF_BIAS, in measure_idle_dispatch()
1760 cycles_to_ns(ce->engine, cycles)); in measure_idle_dispatch()
1762 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_idle_dispatch()
1765 intel_gt_set_wedged(ce->engine->gt); in measure_idle_dispatch()
1774 u32 *cs; in measure_busy_dispatch() local
1787 * Submission latency: B - A in measure_busy_dispatch()
1799 cs = intel_ring_begin(rq, 12); in measure_busy_dispatch()
1800 if (IS_ERR(cs)) { in measure_busy_dispatch()
1802 err = PTR_ERR(cs); in measure_busy_dispatch()
1806 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); in measure_busy_dispatch()
1807 cs = emit_semaphore_poll_until(cs, offset, i); in measure_busy_dispatch()
1808 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_busy_dispatch()
1810 intel_ring_advance(rq, cs); in measure_busy_dispatch()
1812 if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) { in measure_busy_dispatch()
1813 err = -EIO; in measure_busy_dispatch()
1819 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_busy_dispatch()
1822 semaphore_set(sema, i - 1); in measure_busy_dispatch()
1826 wait_for(READ_ONCE(sema[i - 1]), 500); in measure_busy_dispatch()
1827 semaphore_set(sema, i - 1); in measure_busy_dispatch()
1830 GEM_BUG_ON(sema[i] == -1); in measure_busy_dispatch()
1831 elapsed[i - 1] = sema[i] - elapsed[i]; in measure_busy_dispatch()
1836 ce->engine->name, cycles >> TF_BIAS, in measure_busy_dispatch()
1837 cycles_to_ns(ce->engine, cycles)); in measure_busy_dispatch()
1839 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_busy_dispatch()
1842 intel_gt_set_wedged(ce->engine->gt); in measure_busy_dispatch()
1849 i915_ggtt_offset(engine->status_page.vma) + in plug()
1852 u32 *cs; in plug() local
1854 rq = i915_request_create(engine->kernel_context); in plug()
1858 cs = intel_ring_begin(rq, 4); in plug()
1859 if (IS_ERR(cs)) { in plug()
1861 return PTR_ERR(cs); in plug()
1864 cs = emit_semaphore_poll(cs, mode, value, offset); in plug()
1866 intel_ring_advance(rq, cs); in plug()
1891 * Request latency: B - A in measure_inter_request()
1894 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); in measure_inter_request()
1901 return -ENOMEM; in measure_inter_request()
1904 intel_engine_flush_submission(ce->engine); in measure_inter_request()
1907 u32 *cs; in measure_inter_request() local
1915 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, in measure_inter_request()
1923 cs = intel_ring_begin(rq, 4); in measure_inter_request()
1924 if (IS_ERR(cs)) { in measure_inter_request()
1926 err = PTR_ERR(cs); in measure_inter_request()
1930 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_inter_request()
1932 intel_ring_advance(rq, cs); in measure_inter_request()
1938 intel_engine_flush_submission(ce->engine); in measure_inter_request()
1942 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_inter_request()
1947 elapsed[i - 1] = sema[i + 1] - sema[i]; in measure_inter_request()
1950 pr_info("%s: inter-request latency %d cycles, %lluns\n", in measure_inter_request()
1951 ce->engine->name, cycles >> TF_BIAS, in measure_inter_request()
1952 cycles_to_ns(ce->engine, cycles)); in measure_inter_request()
1954 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_inter_request()
1961 intel_gt_set_wedged(ce->engine->gt); in measure_inter_request()
1972 u32 *cs; in measure_context_switch() local
1978 * the inter-context setup we require. in measure_context_switch()
1984 * Context switch latency: B - A in measure_context_switch()
1987 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0); in measure_context_switch()
1993 ce, ce->engine->kernel_context in measure_context_switch()
2008 &fence->fence); in measure_context_switch()
2015 cs = intel_ring_begin(rq, 4); in measure_context_switch()
2016 if (IS_ERR(cs)) { in measure_context_switch()
2018 err = PTR_ERR(cs); in measure_context_switch()
2022 cs = emit_timestamp_store(cs, ce, addr); in measure_context_switch()
2025 intel_ring_advance(rq, cs); in measure_context_switch()
2034 intel_engine_flush_submission(ce->engine); in measure_context_switch()
2037 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_context_switch()
2042 elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1]; in measure_context_switch()
2046 ce->engine->name, cycles >> TF_BIAS, in measure_context_switch()
2047 cycles_to_ns(ce->engine, cycles)); in measure_context_switch()
2049 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_context_switch()
2055 intel_gt_set_wedged(ce->engine->gt); in measure_context_switch()
2064 u32 *cs; in measure_preemption() local
2080 * Preemption dispatch latency: B - A in measure_preemption()
2081 * Preemption switch latency: C - B in measure_preemption()
2084 if (!intel_engine_has_preemption(ce->engine)) in measure_preemption()
2097 cs = intel_ring_begin(rq, 12); in measure_preemption()
2098 if (IS_ERR(cs)) { in measure_preemption()
2100 err = PTR_ERR(cs); in measure_preemption()
2104 cs = emit_store_dw(cs, addr, -1); in measure_preemption()
2105 cs = emit_semaphore_poll_until(cs, offset, i); in measure_preemption()
2106 cs = emit_timestamp_store(cs, ce, addr + sizeof(u32)); in measure_preemption()
2108 intel_ring_advance(rq, cs); in measure_preemption()
2111 if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) { in measure_preemption()
2112 err = -EIO; in measure_preemption()
2116 rq = i915_request_create(ce->engine->kernel_context); in measure_preemption()
2122 cs = intel_ring_begin(rq, 8); in measure_preemption()
2123 if (IS_ERR(cs)) { in measure_preemption()
2125 err = PTR_ERR(cs); in measure_preemption()
2129 cs = emit_timestamp_store(cs, ce, addr); in measure_preemption()
2130 cs = emit_store_dw(cs, offset, i); in measure_preemption()
2132 intel_ring_advance(rq, cs); in measure_preemption()
2133 rq->sched.attr.priority = I915_PRIORITY_BARRIER; in measure_preemption()
2135 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_preemption()
2139 if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) { in measure_preemption()
2140 err = -EIO; in measure_preemption()
2145 elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1]; in measure_preemption()
2149 ce->engine->name, cycles >> TF_BIAS, in measure_preemption()
2150 cycles_to_ns(ce->engine, cycles)); in measure_preemption()
2153 elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0]; in measure_preemption()
2157 ce->engine->name, cycles >> TF_BIAS, in measure_preemption()
2158 cycles_to_ns(ce->engine, cycles)); in measure_preemption()
2160 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_preemption()
2163 intel_gt_set_wedged(ce->engine->gt); in measure_preemption()
2176 smp_store_mb(s->seen, true); /* be safe, be strong */ in signal_cb()
2184 u32 *cs; in measure_completion() local
2196 * Completion latency: B - A in measure_completion()
2209 cs = intel_ring_begin(rq, 12); in measure_completion()
2210 if (IS_ERR(cs)) { in measure_completion()
2212 err = PTR_ERR(cs); in measure_completion()
2216 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1); in measure_completion()
2217 cs = emit_semaphore_poll_until(cs, offset, i); in measure_completion()
2218 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32)); in measure_completion()
2220 intel_ring_advance(rq, cs); in measure_completion()
2222 dma_fence_add_callback(&rq->fence, &cb.base, signal_cb); in measure_completion()
2228 if (wait_for(READ_ONCE(sema[i]) == -1, 50)) { in measure_completion()
2229 err = -EIO; in measure_completion()
2238 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP); in measure_completion()
2242 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2); in measure_completion()
2247 GEM_BUG_ON(sema[i + 1] == -1); in measure_completion()
2248 elapsed[i] = elapsed[i] - sema[i + 1]; in measure_completion()
2253 ce->engine->name, cycles >> TF_BIAS, in measure_completion()
2254 cycles_to_ns(ce->engine, cycles)); in measure_completion()
2256 return intel_gt_wait_for_idle(ce->engine->gt, HZ); in measure_completion()
2259 intel_gt_set_wedged(ce->engine->gt); in measure_completion()
2266 atomic_inc(>->rps.num_waiters); in rps_pin()
2267 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in rps_pin()
2269 mutex_lock(>->rps.lock); in rps_pin()
2270 intel_rps_set(>->rps, gt->rps.max_freq); in rps_pin()
2271 mutex_unlock(>->rps.lock); in rps_pin()
2276 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in rps_unpin()
2277 atomic_dec(>->rps.num_waiters); in rps_unpin()
2287 if (INTEL_GEN(i915) < 8) /* per-engine CS timestamp, semaphores */ in perf_request_latency()
2298 goto out; in perf_request_latency()
2304 goto out; in perf_request_latency()
2308 rps_pin(engine->gt); in perf_request_latency()
2325 rps_unpin(engine->gt); in perf_request_latency()
2331 goto out; in perf_request_latency()
2334 out: in perf_request_latency()
2336 err = -EIO; in perf_request_latency()
2349 GEM_BUG_ON(!ps->nengines); in s_sync0()
2353 rq = i915_request_create(ps->ce[idx]); in s_sync0()
2363 err = -ETIME; in s_sync0()
2368 if (++idx == ps->nengines) in s_sync0()
2383 GEM_BUG_ON(!ps->nengines); in s_sync1()
2387 rq = i915_request_create(ps->ce[idx]); in s_sync1()
2397 err = -ETIME; in s_sync1()
2403 if (++idx == ps->nengines) in s_sync1()
2417 GEM_BUG_ON(!ps->nengines); in s_many()
2421 rq = i915_request_create(ps->ce[idx]); in s_many()
2427 if (++idx == ps->nengines) in s_many()
2454 return -ENOMEM; in perf_series_engines()
2459 return -ENOMEM; in perf_series_engines()
2464 ps->i915 = i915; in perf_series_engines()
2465 ps->nengines = nengines; in perf_series_engines()
2474 goto out; in perf_series_engines()
2480 goto out; in perf_series_engines()
2483 ps->ce[idx++] = ce; in perf_series_engines()
2485 GEM_BUG_ON(idx != ps->nengines); in perf_series_engines()
2499 struct intel_context *ce = ps->ce[idx]; in perf_series_engines()
2501 p->engine = ps->ce[idx]->engine; in perf_series_engines()
2502 intel_engine_pm_get(p->engine); in perf_series_engines()
2504 if (intel_engine_supports_stats(p->engine)) in perf_series_engines()
2505 p->busy = intel_engine_get_busy_time(p->engine, in perf_series_engines()
2506 &p->time) + 1; in perf_series_engines()
2508 p->time = ktime_get(); in perf_series_engines()
2509 p->runtime = -intel_context_get_total_runtime_ns(ce); in perf_series_engines()
2514 err = -EIO; in perf_series_engines()
2518 struct intel_context *ce = ps->ce[idx]; in perf_series_engines()
2522 if (p->busy) in perf_series_engines()
2523 p->busy = ktime_sub(intel_engine_get_busy_time(p->engine, in perf_series_engines()
2525 p->busy - 1); in perf_series_engines()
2528 p->time = ktime_sub(now, p->time); in perf_series_engines()
2531 p->runtime += intel_context_get_total_runtime_ns(ce); in perf_series_engines()
2532 intel_engine_pm_put(p->engine); in perf_series_engines()
2534 busy = 100 * ktime_to_ns(p->busy); in perf_series_engines()
2535 dt = ktime_to_ns(p->time); in perf_series_engines()
2538 busy -= integer * dt; in perf_series_engines()
2546 name, p->engine->name, ce->timeline->seqno, in perf_series_engines()
2548 div_u64(p->runtime, 1000 * 1000), in perf_series_engines()
2549 div_u64(ktime_to_ns(p->time), 1000 * 1000)); in perf_series_engines()
2553 out: in perf_series_engines()
2555 if (IS_ERR_OR_NULL(ps->ce[idx])) in perf_series_engines()
2558 intel_context_unpin(ps->ce[idx]); in perf_series_engines()
2559 intel_context_put(ps->ce[idx]); in perf_series_engines()
2571 struct intel_engine_cs *engine = p->engine; in p_sync0()
2589 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_sync0()
2592 p->time = ktime_get(); in p_sync0()
2611 err = -ETIME; in p_sync0()
2622 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_sync0()
2623 p->busy); in p_sync0()
2624 p->time = ktime_sub(now, p->time); in p_sync0()
2626 p->time = ktime_sub(ktime_get(), p->time); in p_sync0()
2630 p->runtime = intel_context_get_total_runtime_ns(ce); in p_sync0()
2631 p->count = count; in p_sync0()
2641 struct intel_engine_cs *engine = p->engine; in p_sync1()
2660 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_sync1()
2663 p->time = ktime_get(); in p_sync1()
2682 err = -ETIME; in p_sync1()
2695 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_sync1()
2696 p->busy); in p_sync1()
2697 p->time = ktime_sub(now, p->time); in p_sync1()
2699 p->time = ktime_sub(ktime_get(), p->time); in p_sync1()
2703 p->runtime = intel_context_get_total_runtime_ns(ce); in p_sync1()
2704 p->count = count; in p_sync1()
2714 struct intel_engine_cs *engine = p->engine; in p_many()
2732 p->busy = intel_engine_get_busy_time(engine, &p->time); in p_many()
2735 p->time = ktime_get(); in p_many()
2756 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now), in p_many()
2757 p->busy); in p_many()
2758 p->time = ktime_sub(now, p->time); in p_many()
2760 p->time = ktime_sub(ktime_get(), p->time); in p_many()
2764 p->runtime = intel_context_get_total_runtime_ns(ce); in p_many()
2765 p->count = count; in p_many()
2793 return -ENOMEM; in perf_parallel_engines()
2807 atomic_set(&i915->selftest.counter, nengines); in perf_parallel_engines()
2817 "igt:%s", engine->name); in perf_parallel_engines()
2844 err = -EIO; in perf_parallel_engines()
2851 u64 busy = 100 * ktime_to_ns(p->busy); in perf_parallel_engines()
2852 u64 dt = ktime_to_ns(p->time); in perf_parallel_engines()
2857 busy -= integer * dt; in perf_parallel_engines()
2864 GEM_BUG_ON(engine != p->engine); in perf_parallel_engines()
2866 name, engine->name, p->count, integer, decimal, in perf_parallel_engines()
2867 div_u64(p->runtime, 1000 * 1000), in perf_parallel_engines()
2868 div_u64(ktime_to_ns(p->time), 1000 * 1000)); in perf_parallel_engines()
2886 if (intel_gt_is_wedged(&i915->gt)) in i915_request_perf_selftests()