Lines Matching +full:cs +full:- +full:out
2 * SPDX-License-Identifier: MIT
4 * Copyright © 2017-2018 Intel Corporation
26 struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; in hwsp_page()
29 return sg_page(obj->mm.pages->sgl); in hwsp_page()
36 return (address + tl->hwsp_offset) / CACHELINE_BYTES; in hwsp_cacheline()
57 tl = xchg(&state->history[idx], tl); in __mock_hwsp_record()
59 radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); in __mock_hwsp_record()
71 while (count--) { in __mock_hwsp_timeline()
75 tl = intel_timeline_create(state->gt); in __mock_hwsp_timeline()
80 err = radix_tree_insert(&state->cachelines, cacheline, tl); in __mock_hwsp_timeline()
82 if (err == -EEXIST) { in __mock_hwsp_timeline()
90 idx = state->count++ % state->max; in __mock_hwsp_timeline()
95 i915_prandom_shuffle(state->history, in __mock_hwsp_timeline()
96 sizeof(*state->history), in __mock_hwsp_timeline()
97 min(state->count, state->max), in __mock_hwsp_timeline()
98 &state->prng); in __mock_hwsp_timeline()
100 count = i915_prandom_u32_max_state(min(state->count, state->max), in __mock_hwsp_timeline()
101 &state->prng); in __mock_hwsp_timeline()
102 while (count--) { in __mock_hwsp_timeline()
103 idx = --state->count % state->max; in __mock_hwsp_timeline()
127 return -ENOMEM; in mock_hwsp_freelist()
132 state.gt = &i915->gt; in mock_hwsp_freelist()
143 err = -ENOMEM; in mock_hwsp_freelist()
147 for (p = phases; p->name; p++) { in mock_hwsp_freelist()
148 pr_debug("%s(%s)\n", __func__, p->name); in mock_hwsp_freelist()
150 err = __mock_hwsp_timeline(&state, na, p->flags); in mock_hwsp_freelist()
152 goto out; in mock_hwsp_freelist()
156 out: in mock_hwsp_freelist()
179 if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) { in __igt_sync()
181 name, p->name, ctx, p->seqno, yesno(p->expected)); in __igt_sync()
182 return -EINVAL; in __igt_sync()
185 if (p->set) { in __igt_sync()
186 ret = __intel_timeline_sync_set(tl, ctx, p->seqno); in __igt_sync()
206 { "INT_MAX-1", INT_MAX-1, true, false }, in igt_sync()
216 int ret = -ENODEV; in igt_sync()
219 for (p = pass; p->name; p++) { in igt_sync()
221 for (offset = -1; offset <= (order > 1); offset++) { in igt_sync()
226 goto out; in igt_sync()
234 for (offset = -1; offset <= (order > 1); offset++) { in igt_sync()
237 for (p = pass; p->name; p++) { in igt_sync()
240 goto out; in igt_sync()
245 out: in igt_sync()
267 * and the loop itself becomes a significant factor in the per-iteration in bench_sync()
308 while (end_time--) { in bench_sync()
314 return -EINVAL; in bench_sync()
335 pr_info("%s: %lu in-order insertions, %lluns/insert\n", in bench_sync()
341 while (end_time--) { in bench_sync()
345 return -EINVAL; in bench_sync()
349 pr_info("%s: %lu in-order lookups, %lluns/lookup\n", in bench_sync()
381 unsigned int mask = BIT(order) - 1; in bench_sync()
390 * implementation, try to identify its phase-changes in bench_sync()
424 u32 *cs; in emit_ggtt_store_dw() local
426 cs = intel_ring_begin(rq, 4); in emit_ggtt_store_dw()
427 if (IS_ERR(cs)) in emit_ggtt_store_dw()
428 return PTR_ERR(cs); in emit_ggtt_store_dw()
430 if (INTEL_GEN(rq->engine->i915) >= 8) { in emit_ggtt_store_dw()
431 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_ggtt_store_dw()
432 *cs++ = addr; in emit_ggtt_store_dw()
433 *cs++ = 0; in emit_ggtt_store_dw()
434 *cs++ = value; in emit_ggtt_store_dw()
435 } else if (INTEL_GEN(rq->engine->i915) >= 4) { in emit_ggtt_store_dw()
436 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_ggtt_store_dw()
437 *cs++ = 0; in emit_ggtt_store_dw()
438 *cs++ = addr; in emit_ggtt_store_dw()
439 *cs++ = value; in emit_ggtt_store_dw()
441 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; in emit_ggtt_store_dw()
442 *cs++ = addr; in emit_ggtt_store_dw()
443 *cs++ = value; in emit_ggtt_store_dw()
444 *cs++ = MI_NOOP; in emit_ggtt_store_dw()
447 intel_ring_advance(rq, cs); in emit_ggtt_store_dw()
461 goto out; in tl_write()
470 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); in tl_write()
479 out: in tl_write()
494 if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) { in checked_intel_timeline_create()
496 *tl->hwsp_seqno, tl->seqno); in checked_intel_timeline_create()
498 return ERR_PTR(-EINVAL); in checked_intel_timeline_create()
523 return -ENOMEM; in live_hwsp_engine()
558 if (igt_flush_test(gt->i915)) in live_hwsp_engine()
559 err = -EIO; in live_hwsp_engine()
564 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { in live_hwsp_engine()
566 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_engine()
568 err = -EINVAL; in live_hwsp_engine()
598 return -ENOMEM; in live_hwsp_alternate()
612 goto out; in live_hwsp_alternate()
621 goto out; in live_hwsp_alternate()
629 out: in live_hwsp_alternate()
630 if (igt_flush_test(gt->i915)) in live_hwsp_alternate()
631 err = -EIO; in live_hwsp_alternate()
636 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { in live_hwsp_alternate()
638 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_alternate()
640 err = -EINVAL; in live_hwsp_alternate()
667 if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline) in live_hwsp_wrap()
685 goto out; in live_hwsp_wrap()
688 tl->seqno = -4u; in live_hwsp_wrap()
690 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); in live_hwsp_wrap()
692 mutex_unlock(&tl->mutex); in live_hwsp_wrap()
695 goto out; in live_hwsp_wrap()
698 seqno[0], tl->hwsp_offset); in live_hwsp_wrap()
700 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]); in live_hwsp_wrap()
703 goto out; in live_hwsp_wrap()
705 hwsp_seqno[0] = tl->hwsp_seqno; in live_hwsp_wrap()
707 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); in live_hwsp_wrap()
709 mutex_unlock(&tl->mutex); in live_hwsp_wrap()
712 goto out; in live_hwsp_wrap()
715 seqno[1], tl->hwsp_offset); in live_hwsp_wrap()
717 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]); in live_hwsp_wrap()
720 goto out; in live_hwsp_wrap()
722 hwsp_seqno[1] = tl->hwsp_seqno; in live_hwsp_wrap()
731 pr_err("Wait for timeline writes timed out!\n"); in live_hwsp_wrap()
732 err = -EIO; in live_hwsp_wrap()
733 goto out; in live_hwsp_wrap()
741 err = -EINVAL; in live_hwsp_wrap()
742 goto out; in live_hwsp_wrap()
748 out: in live_hwsp_wrap()
749 if (igt_flush_test(gt->i915)) in live_hwsp_wrap()
750 err = -EIO; in live_hwsp_wrap()
771 struct intel_context *ce = engine->kernel_context; in live_hwsp_rollover_kernel()
772 struct intel_timeline *tl = ce->timeline; in live_hwsp_rollover_kernel()
778 err = -EIO; in live_hwsp_rollover_kernel()
779 goto out; in live_hwsp_rollover_kernel()
782 GEM_BUG_ON(i915_active_fence_isset(&tl->last_request)); in live_hwsp_rollover_kernel()
783 tl->seqno = 0; in live_hwsp_rollover_kernel()
786 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); in live_hwsp_rollover_kernel()
794 goto out; in live_hwsp_rollover_kernel()
798 engine->name, in live_hwsp_rollover_kernel()
799 lower_32_bits(this->fence.seqno)); in live_hwsp_rollover_kernel()
801 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); in live_hwsp_rollover_kernel()
808 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); in live_hwsp_rollover_kernel()
811 pr_err("Wait for timeline wrap timed out!\n"); in live_hwsp_rollover_kernel()
812 err = -EIO; in live_hwsp_rollover_kernel()
813 goto out; in live_hwsp_rollover_kernel()
818 pr_err("Pre-wrap request not completed!\n"); in live_hwsp_rollover_kernel()
819 err = -EINVAL; in live_hwsp_rollover_kernel()
820 goto out; in live_hwsp_rollover_kernel()
824 out: in live_hwsp_rollover_kernel()
832 if (igt_flush_test(gt->i915)) in live_hwsp_rollover_kernel()
833 err = -EIO; in live_hwsp_rollover_kernel()
862 goto out; in live_hwsp_rollover_user()
864 tl = ce->timeline; in live_hwsp_rollover_user()
865 if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline) in live_hwsp_rollover_user()
866 goto out; in live_hwsp_rollover_user()
870 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); in live_hwsp_rollover_user()
878 goto out; in live_hwsp_rollover_user()
882 engine->name, in live_hwsp_rollover_user()
883 lower_32_bits(this->fence.seqno)); in live_hwsp_rollover_user()
885 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); in live_hwsp_rollover_user()
892 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); in live_hwsp_rollover_user()
895 pr_err("Wait for timeline wrap timed out!\n"); in live_hwsp_rollover_user()
896 err = -EIO; in live_hwsp_rollover_user()
897 goto out; in live_hwsp_rollover_user()
902 pr_err("Pre-wrap request not completed!\n"); in live_hwsp_rollover_user()
903 err = -EINVAL; in live_hwsp_rollover_user()
904 goto out; in live_hwsp_rollover_user()
908 out: in live_hwsp_rollover_user()
916 if (igt_flush_test(gt->i915)) in live_hwsp_rollover_user()
917 err = -EIO; in live_hwsp_rollover_user()
963 pr_err("Wait for timeline writes timed out!\n"); in live_hwsp_recycle()
966 err = -EIO; in live_hwsp_recycle()
970 if (READ_ONCE(*tl->hwsp_seqno) != count) { in live_hwsp_recycle()
972 count, tl->fence_context, in live_hwsp_recycle()
973 tl->hwsp_offset, *tl->hwsp_seqno); in live_hwsp_recycle()
975 err = -EINVAL; in live_hwsp_recycle()
1005 if (intel_gt_is_wedged(&i915->gt)) in intel_timeline_live_selftests()
1008 return intel_gt_live_subtests(tests, &i915->gt); in intel_timeline_live_selftests()