Lines Matching full:sdma
35 #include "sdma/sdma_4_4_2_offset.h"
36 #include "sdma/sdma_4_4_2_sh_mask.h"
164 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_inst_init_golden_registers()
193 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_init_microcode()
349 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); in sdma_v4_4_2_ring_insert_nop() local
353 if (sdma && sdma->burst_nop && (i == 0)) in sdma_v4_4_2_ring_insert_nop()
430 << (ring->me % adev->sdma.num_inst_per_aid); in sdma_v4_4_2_ring_emit_hdp_flush()
490 struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES]; in sdma_v4_4_2_inst_gfx_stop() local
496 sdma[i] = &adev->sdma.instance[i].ring; in sdma_v4_4_2_inst_gfx_stop()
505 if (sdma[i]->use_doorbell) { in sdma_v4_4_2_inst_gfx_stop()
632 if (adev->sdma.has_page_queue) in sdma_v4_4_2_inst_enable()
635 /* SDMA FW needs to respond to FREEZE requests during reset. in sdma_v4_4_2_inst_enable()
681 struct amdgpu_ring *ring = &adev->sdma.instance[i].ring; in sdma_v4_4_2_gfx_resume()
716 if (adev->sdma.instance[i].gfx_guilty) in sdma_v4_4_2_gfx_resume()
787 struct amdgpu_ring *ring = &adev->sdma.instance[i].page; in sdma_v4_4_2_page_resume()
804 if (adev->sdma.instance[i].page_guilty) in sdma_v4_4_2_page_resume()
905 * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
925 if (!adev->sdma.instance[i].fw) in sdma_v4_4_2_inst_load_microcode()
928 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; in sdma_v4_4_2_inst_load_microcode()
933 (adev->sdma.instance[i].fw->data + in sdma_v4_4_2_inst_load_microcode()
943 adev->sdma.instance[i].fw_version); in sdma_v4_4_2_inst_load_microcode()
970 /* bypass sdma microcode loading on Gopher */ in sdma_v4_4_2_inst_start()
972 adev->sdma.instance[0].fw) { in sdma_v4_4_2_inst_start()
980 /* enable sdma ring preemption */ in sdma_v4_4_2_inst_start()
991 if (adev->sdma.has_page_queue) in sdma_v4_4_2_inst_start()
1024 ring = &adev->sdma.instance[i].ring; in sdma_v4_4_2_inst_start()
1030 if (adev->sdma.has_page_queue) { in sdma_v4_4_2_inst_start()
1031 struct amdgpu_ring *page = &adev->sdma.instance[i].page; in sdma_v4_4_2_inst_start()
1172 * Update PTEs by copying them from the GART using sDMA.
1200 * Update PTEs by writing them manually using sDMA.
1221 * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1230 * Update the page tables using sDMA.
1258 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); in sdma_v4_4_2_ring_pad_ib() local
1264 if (sdma && sdma->burst_nop && (i == 0)) in sdma_v4_4_2_ring_pad_ib()
1295 * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1302 * using sDMA.
1347 adev->sdma.has_page_queue = true; in sdma_v4_4_2_early_init()
1377 * initialization and capability setup are completed before we check the SDMA in sdma_v4_4_2_late_init()
1394 /* SDMA trap event */ in sdma_v4_4_2_sw_init()
1395 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) { in sdma_v4_4_2_sw_init()
1398 &adev->sdma.trap_irq); in sdma_v4_4_2_sw_init()
1403 /* SDMA SRAM ECC event */ in sdma_v4_4_2_sw_init()
1404 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) { in sdma_v4_4_2_sw_init()
1407 &adev->sdma.ecc_irq); in sdma_v4_4_2_sw_init()
1412 /* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/ in sdma_v4_4_2_sw_init()
1413 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) { in sdma_v4_4_2_sw_init()
1416 &adev->sdma.vm_hole_irq); in sdma_v4_4_2_sw_init()
1422 &adev->sdma.doorbell_invalid_irq); in sdma_v4_4_2_sw_init()
1428 &adev->sdma.pool_timeout_irq); in sdma_v4_4_2_sw_init()
1434 &adev->sdma.srbm_write_irq); in sdma_v4_4_2_sw_init()
1440 &adev->sdma.ctxt_empty_irq); in sdma_v4_4_2_sw_init()
1445 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_sw_init()
1446 mutex_init(&adev->sdma.instance[i].engine_reset_mutex); in sdma_v4_4_2_sw_init()
1448 adev->sdma.instance[i].gfx_guilty = false; in sdma_v4_4_2_sw_init()
1449 adev->sdma.instance[i].page_guilty = false; in sdma_v4_4_2_sw_init()
1451 ring = &adev->sdma.instance[i].ring; in sdma_v4_4_2_sw_init()
1454 aid_id = adev->sdma.instance[i].aid_id; in sdma_v4_4_2_sw_init()
1456 DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i, in sdma_v4_4_2_sw_init()
1463 sprintf(ring->name, "sdma%d.%d", aid_id, in sdma_v4_4_2_sw_init()
1464 i % adev->sdma.num_inst_per_aid); in sdma_v4_4_2_sw_init()
1465 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq, in sdma_v4_4_2_sw_init()
1471 if (adev->sdma.has_page_queue) { in sdma_v4_4_2_sw_init()
1472 ring = &adev->sdma.instance[i].page; in sdma_v4_4_2_sw_init()
1484 i % adev->sdma.num_inst_per_aid); in sdma_v4_4_2_sw_init()
1486 &adev->sdma.trap_irq, in sdma_v4_4_2_sw_init()
1494 adev->sdma.supported_reset = in sdma_v4_4_2_sw_init()
1495 amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); in sdma_v4_4_2_sw_init()
1498 dev_err(adev->dev, "fail to initialize sdma ras block\n"); in sdma_v4_4_2_sw_init()
1502 /* Allocate memory for SDMA IP Dump buffer */ in sdma_v4_4_2_sw_init()
1503 ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); in sdma_v4_4_2_sw_init()
1505 adev->sdma.ip_dump = ptr; in sdma_v4_4_2_sw_init()
1507 DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n"); in sdma_v4_4_2_sw_init()
1521 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_sw_fini()
1522 amdgpu_ring_fini(&adev->sdma.instance[i].ring); in sdma_v4_4_2_sw_fini()
1523 if (adev->sdma.has_page_queue) in sdma_v4_4_2_sw_fini()
1524 amdgpu_ring_fini(&adev->sdma.instance[i].page); in sdma_v4_4_2_sw_fini()
1535 kfree(adev->sdma.ip_dump); in sdma_v4_4_2_sw_fini()
1546 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); in sdma_v4_4_2_hw_init()
1564 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); in sdma_v4_4_2_hw_fini()
1566 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_hw_fini()
1567 amdgpu_irq_put(adev, &adev->sdma.ecc_irq, in sdma_v4_4_2_hw_fini()
1601 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_is_idle()
1614 u32 sdma[AMDGPU_MAX_SDMA_INSTANCES]; in sdma_v4_4_2_wait_for_idle() local
1618 for (j = 0; j < adev->sdma.num_instances; j++) { in sdma_v4_4_2_wait_for_idle()
1619 sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG); in sdma_v4_4_2_wait_for_idle()
1620 if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK)) in sdma_v4_4_2_wait_for_idle()
1623 if (j == adev->sdma.num_instances) in sdma_v4_4_2_wait_for_idle()
1659 if (!adev->sdma.has_page_queue) in sdma_v4_4_2_page_ring_is_guilty()
1671 if (!(adev->sdma.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)) in sdma_v4_4_2_reset_queue()
1685 struct amdgpu_ring *ring = &adev->sdma.instance[instance_id].ring; in sdma_v4_4_2_stop_queue()
1691 adev->sdma.instance[instance_id].gfx_guilty = in sdma_v4_4_2_stop_queue()
1693 if (adev->sdma.has_page_queue) in sdma_v4_4_2_stop_queue()
1694 adev->sdma.instance[instance_id].page_guilty = in sdma_v4_4_2_stop_queue()
1703 if (adev->sdma.has_page_queue) { in sdma_v4_4_2_stop_queue()
1704 struct amdgpu_ring *page_ring = &adev->sdma.instance[instance_id].page; in sdma_v4_4_2_stop_queue()
1712 if (adev->sdma.has_page_queue) in sdma_v4_4_2_stop_queue()
1722 struct amdgpu_ring *ring = &adev->sdma.instance[instance_id].ring; in sdma_v4_4_2_restore_queue()
1734 dev_err(adev->dev, "timed out waiting for SDMA%d unhalt after reset\n", in sdma_v4_4_2_restore_queue()
1773 DRM_DEBUG("IH: SDMA trap\n"); in sdma_v4_4_2_process_trap_irq()
1776 /* Client id gives the SDMA instance in AID. To know the exact SDMA in sdma_v4_4_2_process_trap_irq()
1778 * Match node id with the AID id associated with the SDMA instance. */ in sdma_v4_4_2_process_trap_irq()
1779 for (i = instance; i < adev->sdma.num_instances; in sdma_v4_4_2_process_trap_irq()
1780 i += adev->sdma.num_inst_per_aid) { in sdma_v4_4_2_process_trap_irq()
1781 if (adev->sdma.instance[i].aid_id == in sdma_v4_4_2_process_trap_irq()
1786 if (i >= adev->sdma.num_instances) { in sdma_v4_4_2_process_trap_irq()
1789 "Couldn't find the right sdma instance in trap handler"); in sdma_v4_4_2_process_trap_irq()
1795 amdgpu_fence_process(&adev->sdma.instance[i].ring); in sdma_v4_4_2_process_trap_irq()
1798 amdgpu_fence_process(&adev->sdma.instance[i].page); in sdma_v4_4_2_process_trap_irq()
1837 DRM_ERROR("Illegal instruction in SDMA command stream\n"); in sdma_v4_4_2_process_illegal_inst_irq()
1845 drm_sched_fault(&adev->sdma.instance[instance].ring.sched); in sdma_v4_4_2_process_illegal_inst_irq()
1874 if (instance < 0 || instance >= adev->sdma.num_instances) { in sdma_v4_4_2_print_iv_entry()
1875 dev_err(adev->dev, "sdma instance invalid %d\n", instance); in sdma_v4_4_2_print_iv_entry()
1883 "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n", in sdma_v4_4_2_print_iv_entry()
1912 dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n"); in sdma_v4_4_2_process_doorbell_invalid_irq()
1932 "SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n"); in sdma_v4_4_2_process_srbm_write_irq()
1942 dev_dbg_ratelimited(adev->dev, "SDMA context empty interrupt"); in sdma_v4_4_2_process_ctxt_empty_irq()
1959 /* 1-not override: enable sdma mem light sleep */ in sdma_v4_4_2_inst_update_medium_grain_light_sleep()
1967 /* 0-override:disable sdma mem light sleep */ in sdma_v4_4_2_inst_update_medium_grain_light_sleep()
2022 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); in sdma_v4_4_2_set_clockgating_state()
2063 if (!adev->sdma.ip_dump) in sdma_v4_4_2_print_ip_state()
2066 drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances); in sdma_v4_4_2_print_ip_state()
2067 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_print_ip_state()
2073 adev->sdma.ip_dump[instance_offset + j]); in sdma_v4_4_2_print_ip_state()
2084 if (!adev->sdma.ip_dump) in sdma_v4_4_2_dump_ip_state()
2087 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_dump_ip_state()
2090 adev->sdma.ip_dump[instance_offset + j] = in sdma_v4_4_2_dump_ip_state()
2186 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_set_ring_funcs()
2187 adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs; in sdma_v4_4_2_set_ring_funcs()
2188 adev->sdma.instance[i].ring.me = i; in sdma_v4_4_2_set_ring_funcs()
2189 if (adev->sdma.has_page_queue) { in sdma_v4_4_2_set_ring_funcs()
2190 adev->sdma.instance[i].page.funcs = in sdma_v4_4_2_set_ring_funcs()
2192 adev->sdma.instance[i].page.me = i; in sdma_v4_4_2_set_ring_funcs()
2196 /* AID to which SDMA belongs depends on physical instance */ in sdma_v4_4_2_set_ring_funcs()
2197 adev->sdma.instance[i].aid_id = in sdma_v4_4_2_set_ring_funcs()
2198 dev_inst / adev->sdma.num_inst_per_aid; in sdma_v4_4_2_set_ring_funcs()
2238 adev->sdma.trap_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2239 adev->sdma.ecc_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2240 adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2241 adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2242 adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2243 adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2244 adev->sdma.ctxt_empty_irq.num_types = adev->sdma.num_instances; in sdma_v4_4_2_set_irq_funcs()
2246 adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2247 adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2248 adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2249 adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2250 adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2251 adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2252 adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2253 adev->sdma.ctxt_empty_irq.funcs = &sdma_v4_4_2_ctxt_empty_irq_funcs; in sdma_v4_4_2_set_irq_funcs()
2257 * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
2287 * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
2321 if (adev->sdma.has_page_queue) in sdma_v4_4_2_set_buffer_funcs()
2322 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; in sdma_v4_4_2_set_buffer_funcs()
2324 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; in sdma_v4_4_2_set_buffer_funcs()
2341 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v4_4_2_set_vm_pte_funcs()
2342 if (adev->sdma.has_page_queue) in sdma_v4_4_2_set_vm_pte_funcs()
2343 sched = &adev->sdma.instance[i].page.sched; in sdma_v4_4_2_set_vm_pte_funcs()
2345 sched = &adev->sdma.instance[i].ring.sched; in sdma_v4_4_2_set_vm_pte_funcs()
2348 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances; in sdma_v4_4_2_set_vm_pte_funcs()
2352 * sdma_v4_4_2_update_reset_mask - update reset mask for SDMA
2355 * This function update reset mask for SDMA and sets the supported
2366 * the user queue relies on MEC fw and pmfw when the sdma queue do reset. in sdma_v4_4_2_update_reset_mask()
2373 adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; in sdma_v4_4_2_update_reset_mask()
2412 amdgpu_irq_put(adev, &adev->sdma.ecc_irq, in sdma_v4_4_2_xcp_suspend()
2430 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2469 .die_id = adev->sdma.instance[sdma_inst].aid_id, in sdma_v4_4_2_inst_query_ras_error_count()
2472 /* sdma v4_4_2 doesn't support query ce counts */ in sdma_v4_4_2_inst_query_ras_error_count()
2491 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); in sdma_v4_4_2_query_ras_error_count()
2496 dev_warn(adev->dev, "SDMA RAS is not supported\n"); in sdma_v4_4_2_query_ras_error_count()
2516 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); in sdma_v4_4_2_reset_ras_error_count()
2521 dev_warn(adev->dev, "SDMA RAS is not supported\n"); in sdma_v4_4_2_reset_ras_error_count()
2614 adev->sdma.ras = &sdma_v4_4_2_ras; in sdma_v4_4_2_set_ras_funcs()