Lines Matching +full:is +full:- +full:decoded +full:- +full:cs
1 // SPDX-License-Identifier: MIT
46 return gt_to_xe(lrc->fence_ctx.gt); in lrc_to_xe()
86 * The per-platform tables are u8-encoded in @data. Decode @data and set the
87 * addresses' offset and commands in @regs. The following encoding is used
91 * [7]: create NOPs - number of NOPs are set in lower bits
97 * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count"
99 * is used for offsets smaller than 0x200 while the latter is for values bigger
121 const u32 base = hwe->mmio_base; in set_offsets()
142 xe_gt_assert(hwe->gt, count); in set_offsets()
155 } while (--count); in set_offsets()
577 if (xe_gt_has_indirect_ring_state(hwe->gt)) in set_context_control()
586 struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; in set_memory_based_intr()
587 struct xe_device *xe = gt_to_xe(hwe->gt); in set_memory_based_intr()
614 struct xe_device *xe = gt_to_xe(hwe->gt); in lrc_ring_mi_mode()
633 return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; in xe_lrc_has_indirect_ring_state()
643 return lrc->ring.size; in xe_lrc_pphwsp_offset()
677 /* The seqno is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_seqno_offset()
683 /* The start seqno is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_start_seqno_offset()
689 /* This is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_ctx_job_timestamp_offset()
695 /* The parallel is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_parallel_offset()
716 /* Indirect ring state page is at the very end of LRC */ in __xe_lrc_indirect_ring_offset()
717 return lrc->size - LRC_INDIRECT_RING_STATE_SIZE; in __xe_lrc_indirect_ring_offset()
723 struct iosys_map map = lrc->bo->vmap; \
731 return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \
749 * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address in DECL_MAP_ADDR_HELPERS()
760 * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address
771 * xe_lrc_ctx_timestamp() - Read ctx timestamp value
785 if (xe->info.has_64bit_timestamp) { in xe_lrc_ctx_timestamp()
794 * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address
805 * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value
875 struct xe_gt *gt = hwe->gt; in empty_lrc_data()
879 data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); in empty_lrc_data()
883 /* 1st page: Per-Process of HW status Page */ in empty_lrc_data()
885 set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); in empty_lrc_data()
890 regs = data + xe_gt_lrc_size(gt, hwe->class) - in empty_lrc_data()
900 u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); in xe_lrc_set_ppgtt()
908 xe_hw_fence_ctx_finish(&lrc->fence_ctx); in xe_lrc_finish()
909 xe_bo_lock(lrc->bo, false); in xe_lrc_finish()
910 xe_bo_unpin(lrc->bo); in xe_lrc_finish()
911 xe_bo_unlock(lrc->bo); in xe_lrc_finish()
912 xe_bo_put(lrc->bo); in xe_lrc_finish()
913 xe_bo_unpin_map_no_vm(lrc->bb_per_ctx_bo); in xe_lrc_finish()
917 * xe_lrc_setup_utilization() - Setup wa bb to assist in calculating active
926 * (1) Determine if the context is running:
928 * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is
930 * context is active. If the context switched out, it would overwrite this
936 * The CTX_TIMESTAMP ticks only when the context is active. If a context is
939 * engine instance. Since we do not know which instance the context is running
940 * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and
948 cmd = lrc->bb_per_ctx_bo->vmap.vaddr; in xe_lrc_setup_utilization()
960 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) { in xe_lrc_setup_utilization()
970 xe_bo_ggtt_addr(lrc->bb_per_ctx_bo) | 1); in xe_lrc_setup_utilization()
981 struct xe_gt *gt = hwe->gt; in xe_lrc_init()
991 kref_init(&lrc->refcount); in xe_lrc_init()
992 lrc->gt = gt; in xe_lrc_init()
993 lrc->flags = 0; in xe_lrc_init()
994 lrc_size = ring_size + xe_gt_lrc_size(gt, hwe->class); in xe_lrc_init()
996 lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; in xe_lrc_init()
1002 * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address in xe_lrc_init()
1005 lrc->bo = xe_bo_create_pin_map(xe, tile, vm, lrc_size, in xe_lrc_init()
1008 if (IS_ERR(lrc->bo)) in xe_lrc_init()
1009 return PTR_ERR(lrc->bo); in xe_lrc_init()
1011 lrc->bb_per_ctx_bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K, in xe_lrc_init()
1014 if (IS_ERR(lrc->bb_per_ctx_bo)) { in xe_lrc_init()
1015 err = PTR_ERR(lrc->bb_per_ctx_bo); in xe_lrc_init()
1019 lrc->size = lrc_size; in xe_lrc_init()
1020 lrc->ring.size = ring_size; in xe_lrc_init()
1021 lrc->ring.tail = 0; in xe_lrc_init()
1023 xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, in xe_lrc_init()
1024 hwe->fence_irq, hwe->name); in xe_lrc_init()
1026 if (!gt->default_lrc[hwe->class]) { in xe_lrc_init()
1029 err = -ENOMEM; in xe_lrc_init()
1035 * Init Per-Process of HW status Page, LRC / context state to known in xe_lrc_init()
1042 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, in xe_lrc_init()
1043 xe_gt_lrc_size(gt, hwe->class) - LRC_PPHWSP_SIZE); in xe_lrc_init()
1046 xe_gt_lrc_size(gt, hwe->class)); in xe_lrc_init()
1053 if (vm->xef) in xe_lrc_init()
1054 xe_drm_client_add_bo(vm->xef->client, lrc->bo); in xe_lrc_init()
1059 xe_memirq_status_ptr(&tile->memirq, hwe)); in xe_lrc_init()
1061 xe_memirq_source_ptr(&tile->memirq, hwe)); in xe_lrc_init()
1073 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); in xe_lrc_init()
1075 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); in xe_lrc_init()
1079 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); in xe_lrc_init()
1081 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); in xe_lrc_init()
1094 lrc->ctx_timestamp = 0; in xe_lrc_init()
1096 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) in xe_lrc_init()
1099 if (xe->info.has_asid && vm) in xe_lrc_init()
1100 xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, vm->usm.asid); in xe_lrc_init()
1102 lrc->desc = LRC_VALID; in xe_lrc_init()
1103 lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); in xe_lrc_init()
1110 lrc->desc |= LRC_PRIVILEGE; in xe_lrc_init()
1113 lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); in xe_lrc_init()
1114 lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); in xe_lrc_init()
1121 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); in xe_lrc_init()
1124 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); in xe_lrc_init()
1136 * xe_lrc_create - Create a LRC
1140 * @msix_vec: MSI-X interrupt vector (for platforms that support it)
1156 return ERR_PTR(-ENOMEM); in xe_lrc_create()
1168 * xe_lrc_destroy - Destroy the LRC
1225 const u32 tail = lrc->ring.tail; in xe_lrc_ring_space()
1226 const u32 size = lrc->ring.size; in xe_lrc_ring_space()
1228 return ((head - tail - 1) & (size - 1)) + 1; in xe_lrc_ring_space()
1236 iosys_map_incr(&ring, lrc->ring.tail); in __xe_lrc_write_ring()
1238 lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); in __xe_lrc_write_ring()
1253 xe_assert(xe, lrc->ring.tail < lrc->ring.size); in xe_lrc_write_ring()
1254 rhs = lrc->ring.size - lrc->ring.tail; in xe_lrc_write_ring()
1257 __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); in xe_lrc_write_ring()
1271 return lrc->desc | xe_lrc_ggtt_addr(lrc); in xe_lrc_descriptor()
1280 * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence.
1293 * xe_lrc_free_seqno_fence() - Free an lrc seqno fence.
1305 * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence.
1309 * Initializes a pre-allocated lrc seqno fence.
1310 * After initialization, the fence is subject to normal
1311 * dma-fence refcounting.
1315 xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); in xe_lrc_init_seqno_fence()
1348 * xe_lrc_engine_id() - Read engine id value
1369 /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */ in instr_dw()
1417 inst_header, (numdw - 1) / 2); in dump_mi_command()
1419 drm_printf(p, " - %#6x = %#010x\n", dw[i], dw[i + 1]); in dump_mi_command()
1428 drm_printf(p, " - %#6x = %#010llx\n", in dump_mi_command()
1431 drm_printf(p, " - %*ph (%s)\n", in dump_mi_command()
1432 (int)sizeof(u32) * (numdw - 1), dw + 1, in dump_mi_command()
1458 * Make sure we haven't mis-parsed a number of dwords that exceeds the in dump_gfxpipe_command()
1607 * Make sure we haven't mis-parsed a number of dwords that exceeds the in dump_gfx_state_command()
1630 if (!gt->default_lrc[hwe_class]) { in xe_lrc_dump_default()
1636 * Skip the beginning of the LRC since it contains the per-process in xe_lrc_dump_default()
1639 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; in xe_lrc_dump_default()
1640 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; in xe_lrc_dump_default()
1657 remaining_dw -= num_dw; in xe_lrc_dump_default()
1721 struct xe_gt *gt = q->hwe->gt; in xe_lrc_emit_hwe_state_instructions()
1744 * any of the state between contexts, even if that leakage is harmless. in xe_lrc_emit_hwe_state_instructions()
1746 if (XE_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { in xe_lrc_emit_hwe_state_instructions()
1752 xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", in xe_lrc_emit_hwe_state_instructions()
1767 * Xe2's SVG context is the same as the one on DG2 / MTL in xe_lrc_emit_hwe_state_instructions()
1768 * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has in xe_lrc_emit_hwe_state_instructions()
1777 bb->cs[bb->len] = instr; in xe_lrc_emit_hwe_state_instructions()
1779 bb->cs[bb->len] |= (num_dw - 2); in xe_lrc_emit_hwe_state_instructions()
1781 bb->len += num_dw; in xe_lrc_emit_hwe_state_instructions()
1792 if (lrc->bo->vm) in xe_lrc_snapshot_capture()
1793 xe_vm_get(lrc->bo->vm); in xe_lrc_snapshot_capture()
1795 snapshot->context_desc = xe_lrc_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1796 snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1797 snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1798 snapshot->head = xe_lrc_ring_head(lrc); in xe_lrc_snapshot_capture()
1799 snapshot->tail.internal = lrc->ring.tail; in xe_lrc_snapshot_capture()
1800 snapshot->tail.memory = xe_lrc_ring_tail(lrc); in xe_lrc_snapshot_capture()
1801 snapshot->start = xe_lrc_ring_start(lrc); in xe_lrc_snapshot_capture()
1802 snapshot->start_seqno = xe_lrc_start_seqno(lrc); in xe_lrc_snapshot_capture()
1803 snapshot->seqno = xe_lrc_seqno(lrc); in xe_lrc_snapshot_capture()
1804 snapshot->lrc_bo = xe_bo_get(lrc->bo); in xe_lrc_snapshot_capture()
1805 snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); in xe_lrc_snapshot_capture()
1806 snapshot->lrc_size = lrc->bo->size - snapshot->lrc_offset; in xe_lrc_snapshot_capture()
1807 snapshot->lrc_snapshot = NULL; in xe_lrc_snapshot_capture()
1808 snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); in xe_lrc_snapshot_capture()
1809 snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); in xe_lrc_snapshot_capture()
1822 bo = snapshot->lrc_bo; in xe_lrc_snapshot_capture_delayed()
1823 vm = bo->vm; in xe_lrc_snapshot_capture_delayed()
1824 snapshot->lrc_bo = NULL; in xe_lrc_snapshot_capture_delayed()
1826 snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); in xe_lrc_snapshot_capture_delayed()
1827 if (!snapshot->lrc_snapshot) in xe_lrc_snapshot_capture_delayed()
1831 if (!ttm_bo_vmap(&bo->ttm, &src)) { in xe_lrc_snapshot_capture_delayed()
1833 snapshot->lrc_snapshot, &src, snapshot->lrc_offset, in xe_lrc_snapshot_capture_delayed()
1834 snapshot->lrc_size); in xe_lrc_snapshot_capture_delayed()
1835 ttm_bo_vunmap(&bo->ttm, &src); in xe_lrc_snapshot_capture_delayed()
1837 kvfree(snapshot->lrc_snapshot); in xe_lrc_snapshot_capture_delayed()
1838 snapshot->lrc_snapshot = NULL; in xe_lrc_snapshot_capture_delayed()
1854 drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); in xe_lrc_snapshot_print()
1856 snapshot->ring_addr); in xe_lrc_snapshot_print()
1858 snapshot->indirect_context_desc); in xe_lrc_snapshot_print()
1859 drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); in xe_lrc_snapshot_print()
1861 snapshot->tail.internal, snapshot->tail.memory); in xe_lrc_snapshot_print()
1862 drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); in xe_lrc_snapshot_print()
1863 drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); in xe_lrc_snapshot_print()
1864 drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); in xe_lrc_snapshot_print()
1865 drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); in xe_lrc_snapshot_print()
1866 drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); in xe_lrc_snapshot_print()
1868 if (!snapshot->lrc_snapshot) in xe_lrc_snapshot_print()
1874 u32 *val = snapshot->lrc_snapshot + i; in xe_lrc_snapshot_print()
1880 drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); in xe_lrc_snapshot_print()
1882 for (; i < snapshot->lrc_size; i += sizeof(u32)) { in xe_lrc_snapshot_print()
1883 u32 *val = snapshot->lrc_snapshot + i; in xe_lrc_snapshot_print()
1896 kvfree(snapshot->lrc_snapshot); in xe_lrc_snapshot_free()
1897 if (snapshot->lrc_bo) { in xe_lrc_snapshot_free()
1900 vm = snapshot->lrc_bo->vm; in xe_lrc_snapshot_free()
1901 xe_bo_put(snapshot->lrc_bo); in xe_lrc_snapshot_free()
1915 hwe = xe_gt_hw_engine(lrc->gt, class, instance, false); in get_ctx_timestamp()
1916 if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe), in get_ctx_timestamp()
1919 return -1; in get_ctx_timestamp()
1921 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) in get_ctx_timestamp()
1922 val = xe_mmio_read64_2x32(&hwe->gt->mmio, in get_ctx_timestamp()
1923 RING_CTX_TIMESTAMP(hwe->mmio_base)); in get_ctx_timestamp()
1925 val = xe_mmio_read32(&hwe->gt->mmio, in get_ctx_timestamp()
1926 RING_CTX_TIMESTAMP(hwe->mmio_base)); in get_ctx_timestamp()
1934 * xe_lrc_update_timestamp() - Update ctx timestamp
1940 * slightly racy, so follow a read-again logic to ensure that the context is
1950 *old_ts = lrc->ctx_timestamp; in xe_lrc_update_timestamp()
1953 /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ in xe_lrc_update_timestamp()
1955 lrc->ctx_timestamp = lrc_ts; in xe_lrc_update_timestamp()
1962 lrc->ctx_timestamp = reg_ts; in xe_lrc_update_timestamp()
1964 /* read lrc again to ensure context is still active */ in xe_lrc_update_timestamp()
1973 lrc->ctx_timestamp = lrc_ts; in xe_lrc_update_timestamp()
1978 return lrc->ctx_timestamp; in xe_lrc_update_timestamp()
1982 * xe_lrc_ring_is_idle() - LRC is idle
1987 * Return: True is ring is idle, False otherwise