Lines Matching +full:disable +full:- +full:eop
889 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v7_0_free_microcode()
890 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v7_0_free_microcode()
891 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v7_0_free_microcode()
892 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v7_0_free_microcode()
893 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v7_0_free_microcode()
894 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v7_0_free_microcode()
901 * gfx_v7_0_init_microcode - load ucode images from disk
916 switch (adev->asic_type) { in gfx_v7_0_init_microcode()
936 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, in gfx_v7_0_init_microcode()
942 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, in gfx_v7_0_init_microcode()
948 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, in gfx_v7_0_init_microcode()
954 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v7_0_init_microcode()
960 if (adev->asic_type == CHIP_KAVERI) { in gfx_v7_0_init_microcode()
961 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v7_0_init_microcode()
968 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, in gfx_v7_0_init_microcode()
980 * gfx_v7_0_tiling_mode_table_init - init the hw tiling table
993 ARRAY_SIZE(adev->gfx.config.tile_mode_array); in gfx_v7_0_tiling_mode_table_init()
995 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); in gfx_v7_0_tiling_mode_table_init()
999 tile = adev->gfx.config.tile_mode_array; in gfx_v7_0_tiling_mode_table_init()
1000 macrotile = adev->gfx.config.macrotile_mode_array; in gfx_v7_0_tiling_mode_table_init()
1002 switch (adev->gfx.config.mem_row_size_in_kb) { in gfx_v7_0_tiling_mode_table_init()
1020 switch (adev->asic_type) { in gfx_v7_0_tiling_mode_table_init()
1545 * gfx_v7_0_select_se_sh - select which SE, SH to address
1582 * gfx_v7_0_get_rb_active_bitmap - computes the mask of enabled RBs
1599 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / in gfx_v7_0_get_rb_active_bitmap()
1600 adev->gfx.config.max_sh_per_se); in gfx_v7_0_get_rb_active_bitmap()
1608 switch (adev->asic_type) { in gfx_v7_0_raster_config()
1632 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); in gfx_v7_0_raster_config()
1642 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); in gfx_v7_0_write_harvested_raster_configs()
1643 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); in gfx_v7_0_write_harvested_raster_configs()
1649 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; in gfx_v7_0_write_harvested_raster_configs()
1673 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); in gfx_v7_0_write_harvested_raster_configs()
1747 * gfx_v7_0_setup_rb - setup the RBs on the asic
1751 * Configures per-SE/SH RB registers (CIK).
1759 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / in gfx_v7_0_setup_rb()
1760 adev->gfx.config.max_sh_per_se; in gfx_v7_0_setup_rb()
1763 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_setup_rb()
1764 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v7_0_setup_rb()
1765 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v7_0_setup_rb()
1768 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * in gfx_v7_0_setup_rb()
1774 adev->gfx.config.backend_enable_mask = active_rbs; in gfx_v7_0_setup_rb()
1775 adev->gfx.config.num_rbs = hweight32(active_rbs); in gfx_v7_0_setup_rb()
1777 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * in gfx_v7_0_setup_rb()
1778 adev->gfx.config.max_shader_engines, 16); in gfx_v7_0_setup_rb()
1782 if (!adev->gfx.config.backend_enable_mask || in gfx_v7_0_setup_rb()
1783 adev->gfx.config.num_rbs >= num_rb_pipes) { in gfx_v7_0_setup_rb()
1788 adev->gfx.config.backend_enable_mask, in gfx_v7_0_setup_rb()
1793 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v7_0_setup_rb()
1794 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v7_0_setup_rb()
1796 adev->gfx.config.rb_config[i][j].rb_backend_disable = in gfx_v7_0_setup_rb()
1798 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = in gfx_v7_0_setup_rb()
1800 adev->gfx.config.rb_config[i][j].raster_config = in gfx_v7_0_setup_rb()
1802 adev->gfx.config.rb_config[i][j].raster_config_1 = in gfx_v7_0_setup_rb()
1807 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_setup_rb()
1812 * gfx_v7_0_init_compute_vmid - gart enable
1827 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) in gfx_v7_0_init_compute_vmid()
1828 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) in gfx_v7_0_init_compute_vmid()
1829 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) in gfx_v7_0_init_compute_vmid()
1835 mutex_lock(&adev->srbm_mutex); in gfx_v7_0_init_compute_vmid()
1836 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v7_0_init_compute_vmid()
1845 mutex_unlock(&adev->srbm_mutex); in gfx_v7_0_init_compute_vmid()
1849 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v7_0_init_compute_vmid()
1862 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA in gfx_v7_0_init_gds_vmid()
1877 adev->gfx.config.double_offchip_lds_buf = 1; in gfx_v7_0_config_init()
1881 * gfx_v7_0_constants_init - setup the 3D engine
1896 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); in gfx_v7_0_constants_init()
1897 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); in gfx_v7_0_constants_init()
1898 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); in gfx_v7_0_constants_init()
1911 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_constants_init()
1936 mutex_lock(&adev->srbm_mutex); in gfx_v7_0_constants_init()
1937 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { in gfx_v7_0_constants_init()
1941 sh_mem_base = adev->gmc.shared_aperture_start >> 48; in gfx_v7_0_constants_init()
1950 mutex_unlock(&adev->srbm_mutex); in gfx_v7_0_constants_init()
1982 …((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIF… in gfx_v7_0_constants_init()
1983 …(adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) | in gfx_v7_0_constants_init()
1984 (adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) | in gfx_v7_0_constants_init()
1985 (adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT))); in gfx_v7_0_constants_init()
2015 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_constants_init()
2021 * gfx_v7_0_ring_test_ring - basic gfx ring test
2032 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_test_ring()
2043 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START); in gfx_v7_0_ring_test_ring()
2047 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_ring_test_ring()
2053 if (i >= adev->usec_timeout) in gfx_v7_0_ring_test_ring()
2054 r = -ETIMEDOUT; in gfx_v7_0_ring_test_ring()
2059 * gfx_v7_0_ring_emit_hdp_flush - emit an hdp flush on the cp
2068 int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1; in gfx_v7_0_ring_emit_hdp_flush()
2070 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { in gfx_v7_0_ring_emit_hdp_flush()
2071 switch (ring->me) { in gfx_v7_0_ring_emit_hdp_flush()
2073 ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; in gfx_v7_0_ring_emit_hdp_flush()
2076 ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; in gfx_v7_0_ring_emit_hdp_flush()
2108 * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring
2125 /* Workaround for cache flush problems. First send a dummy EOP in gfx_v7_0_ring_emit_fence_gfx()
2136 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); in gfx_v7_0_ring_emit_fence_gfx()
2137 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); in gfx_v7_0_ring_emit_fence_gfx()
2139 /* Then send the real EOP event down the pipe. */ in gfx_v7_0_ring_emit_fence_gfx()
2154 * gfx_v7_0_ring_emit_fence_compute - emit a fence on the compute ring
2171 /* RELEASE_MEM - flush caches, send int */ in gfx_v7_0_ring_emit_fence_compute()
2188 * gfx_v7_0_ring_emit_ib_gfx - emit an IB (Indirect Buffer) on the ring
2215 if (ib->flags & AMDGPU_IB_FLAG_CE) in gfx_v7_0_ring_emit_ib_gfx()
2220 control |= ib->length_dw | (vmid << 24); in gfx_v7_0_ring_emit_ib_gfx()
2227 (ib->gpu_addr & 0xFFFFFFFC)); in gfx_v7_0_ring_emit_ib_gfx()
2228 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v7_0_ring_emit_ib_gfx()
2238 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); in gfx_v7_0_ring_emit_ib_compute()
2250 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { in gfx_v7_0_ring_emit_ib_compute()
2252 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); in gfx_v7_0_ring_emit_ib_compute()
2253 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); in gfx_v7_0_ring_emit_ib_compute()
2261 (ib->gpu_addr & 0xFFFFFFFC)); in gfx_v7_0_ring_emit_ib_compute()
2262 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v7_0_ring_emit_ib_compute()
2287 * gfx_v7_0_ring_test_ib - basic ring IB test
2298 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_test_ib()
2311 ib.ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START; in gfx_v7_0_ring_test_ib()
2321 r = -ETIMEDOUT; in gfx_v7_0_ring_test_ib()
2330 r = -EINVAL; in gfx_v7_0_ring_test_ib()
2345 * PFP - Pre-Fetch Parser
2346 * ME - Micro Engine
2347 * CE - Constant Engine
2355 * MEC1 - Compute MicroEngine 1
2356 * MEC2 - Compute MicroEngine 2
2362 * gfx_v7_0_cp_gfx_enable - enable/disable the gfx CP MEs
2365 * @enable: enable or disable the MEs
2381 * gfx_v7_0_cp_gfx_load_microcode - load the gfx CP ME ucode
2386 * Returns 0 for success, -EINVAL if the ucode is not available.
2396 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) in gfx_v7_0_cp_gfx_load_microcode()
2397 return -EINVAL; in gfx_v7_0_cp_gfx_load_microcode()
2399 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; in gfx_v7_0_cp_gfx_load_microcode()
2400 ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; in gfx_v7_0_cp_gfx_load_microcode()
2401 me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; in gfx_v7_0_cp_gfx_load_microcode()
2403 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); in gfx_v7_0_cp_gfx_load_microcode()
2404 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); in gfx_v7_0_cp_gfx_load_microcode()
2405 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); in gfx_v7_0_cp_gfx_load_microcode()
2406 adev->gfx.pfp_fw_version = le32_to_cpu(pfp_hdr->header.ucode_version); in gfx_v7_0_cp_gfx_load_microcode()
2407 adev->gfx.ce_fw_version = le32_to_cpu(ce_hdr->header.ucode_version); in gfx_v7_0_cp_gfx_load_microcode()
2408 adev->gfx.me_fw_version = le32_to_cpu(me_hdr->header.ucode_version); in gfx_v7_0_cp_gfx_load_microcode()
2409 adev->gfx.me_feature_version = le32_to_cpu(me_hdr->ucode_feature_version); in gfx_v7_0_cp_gfx_load_microcode()
2410 adev->gfx.ce_feature_version = le32_to_cpu(ce_hdr->ucode_feature_version); in gfx_v7_0_cp_gfx_load_microcode()
2411 adev->gfx.pfp_feature_version = le32_to_cpu(pfp_hdr->ucode_feature_version); in gfx_v7_0_cp_gfx_load_microcode()
2417 (adev->gfx.pfp_fw->data + in gfx_v7_0_cp_gfx_load_microcode()
2418 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_cp_gfx_load_microcode()
2419 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_cp_gfx_load_microcode()
2423 WREG32(mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); in gfx_v7_0_cp_gfx_load_microcode()
2427 (adev->gfx.ce_fw->data + in gfx_v7_0_cp_gfx_load_microcode()
2428 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_cp_gfx_load_microcode()
2429 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_cp_gfx_load_microcode()
2433 WREG32(mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); in gfx_v7_0_cp_gfx_load_microcode()
2437 (adev->gfx.me_fw->data + in gfx_v7_0_cp_gfx_load_microcode()
2438 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_cp_gfx_load_microcode()
2439 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_cp_gfx_load_microcode()
2443 WREG32(mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); in gfx_v7_0_cp_gfx_load_microcode()
2449 * gfx_v7_0_cp_gfx_start - start the gfx ring
2459 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; in gfx_v7_0_cp_gfx_start()
2465 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); in gfx_v7_0_cp_gfx_start()
2491 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v7_0_cp_gfx_start()
2492 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v7_0_cp_gfx_start()
2493 if (sect->id == SECT_CONTEXT) { in gfx_v7_0_cp_gfx_start()
2495 PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); in gfx_v7_0_cp_gfx_start()
2496 amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START); in gfx_v7_0_cp_gfx_start()
2497 for (i = 0; i < ext->reg_count; i++) in gfx_v7_0_cp_gfx_start()
2498 amdgpu_ring_write(ring, ext->extent[i]); in gfx_v7_0_cp_gfx_start()
2504 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); in gfx_v7_0_cp_gfx_start()
2505 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); in gfx_v7_0_cp_gfx_start()
2506 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); in gfx_v7_0_cp_gfx_start()
2525 * gfx_v7_0_cp_gfx_resume - setup the gfx ring buffer registers
2542 if (adev->asic_type != CHIP_HAWAII) in gfx_v7_0_cp_gfx_resume()
2553 /* ring 0 - compute and gfx */ in gfx_v7_0_cp_gfx_resume()
2555 ring = &adev->gfx.gfx_ring[0]; in gfx_v7_0_cp_gfx_resume()
2556 rb_bufsz = order_base_2(ring->ring_size / 8); in gfx_v7_0_cp_gfx_resume()
2565 ring->wptr = 0; in gfx_v7_0_cp_gfx_resume()
2566 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v7_0_cp_gfx_resume()
2569 rptr_addr = ring->rptr_gpu_addr; in gfx_v7_0_cp_gfx_resume()
2579 rb_addr = ring->gpu_addr >> 8; in gfx_v7_0_cp_gfx_resume()
2594 return *ring->rptr_cpu_addr; in gfx_v7_0_ring_get_rptr()
2599 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_get_wptr_gfx()
2606 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_set_wptr_gfx()
2608 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v7_0_ring_set_wptr_gfx()
2615 return *ring->wptr_cpu_addr; in gfx_v7_0_ring_get_wptr_compute()
2620 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_set_wptr_compute()
2623 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); in gfx_v7_0_ring_set_wptr_compute()
2624 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); in gfx_v7_0_ring_set_wptr_compute()
2628 * gfx_v7_0_cp_compute_enable - enable/disable the compute CP MEs
2631 * @enable: enable or disable the MEs
2646 * gfx_v7_0_cp_compute_load_microcode - load the compute CP ME ucode
2651 * Returns 0 for success, -EINVAL if the ucode is not available.
2659 if (!adev->gfx.mec_fw) in gfx_v7_0_cp_compute_load_microcode()
2660 return -EINVAL; in gfx_v7_0_cp_compute_load_microcode()
2662 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v7_0_cp_compute_load_microcode()
2663 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); in gfx_v7_0_cp_compute_load_microcode()
2664 adev->gfx.mec_fw_version = le32_to_cpu(mec_hdr->header.ucode_version); in gfx_v7_0_cp_compute_load_microcode()
2665 adev->gfx.mec_feature_version = le32_to_cpu( in gfx_v7_0_cp_compute_load_microcode()
2666 mec_hdr->ucode_feature_version); in gfx_v7_0_cp_compute_load_microcode()
2672 (adev->gfx.mec_fw->data + in gfx_v7_0_cp_compute_load_microcode()
2673 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_cp_compute_load_microcode()
2674 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_cp_compute_load_microcode()
2680 if (adev->asic_type == CHIP_KAVERI) { in gfx_v7_0_cp_compute_load_microcode()
2683 if (!adev->gfx.mec2_fw) in gfx_v7_0_cp_compute_load_microcode()
2684 return -EINVAL; in gfx_v7_0_cp_compute_load_microcode()
2686 mec2_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data; in gfx_v7_0_cp_compute_load_microcode()
2687 amdgpu_ucode_print_gfx_hdr(&mec2_hdr->header); in gfx_v7_0_cp_compute_load_microcode()
2688 adev->gfx.mec2_fw_version = le32_to_cpu(mec2_hdr->header.ucode_version); in gfx_v7_0_cp_compute_load_microcode()
2689 adev->gfx.mec2_feature_version = le32_to_cpu( in gfx_v7_0_cp_compute_load_microcode()
2690 mec2_hdr->ucode_feature_version); in gfx_v7_0_cp_compute_load_microcode()
2694 (adev->gfx.mec2_fw->data + in gfx_v7_0_cp_compute_load_microcode()
2695 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_cp_compute_load_microcode()
2696 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_cp_compute_load_microcode()
2707 * gfx_v7_0_cp_compute_fini - stop the compute queues
2718 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v7_0_cp_compute_fini()
2719 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v7_0_cp_compute_fini()
2721 amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL); in gfx_v7_0_cp_compute_fini()
2727 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); in gfx_v7_0_mec_fini()
2736 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in gfx_v7_0_mec_init()
2742 mec_hpd_size = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec in gfx_v7_0_mec_init()
2748 &adev->gfx.mec.hpd_eop_obj, in gfx_v7_0_mec_init()
2749 &adev->gfx.mec.hpd_eop_gpu_addr, in gfx_v7_0_mec_init()
2752 dev_warn(adev->dev, "(%d) create, pin or map of HDP EOP bo failed\n", r); in gfx_v7_0_mec_init()
2760 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); in gfx_v7_0_mec_init()
2761 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); in gfx_v7_0_mec_init()
2771 size_t eop_offset = (mec * adev->gfx.mec.num_pipe_per_mec + pipe) in gfx_v7_0_compute_pipe_init()
2774 mutex_lock(&adev->srbm_mutex); in gfx_v7_0_compute_pipe_init()
2775 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + eop_offset; in gfx_v7_0_compute_pipe_init()
2779 /* write the EOP addr */ in gfx_v7_0_compute_pipe_init()
2786 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ in gfx_v7_0_compute_pipe_init()
2793 mutex_unlock(&adev->srbm_mutex); in gfx_v7_0_compute_pipe_init()
2800 /* disable the queue if it's active */ in gfx_v7_0_mqd_deactivate()
2803 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_mqd_deactivate()
2809 if (i == adev->usec_timeout) in gfx_v7_0_mqd_deactivate()
2810 return -ETIMEDOUT; in gfx_v7_0_mqd_deactivate()
2831 mqd->header = 0xC0310800; in gfx_v7_0_mqd_init()
2832 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in gfx_v7_0_mqd_init()
2833 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in gfx_v7_0_mqd_init()
2834 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in gfx_v7_0_mqd_init()
2835 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in gfx_v7_0_mqd_init()
2838 mqd->cp_hqd_pq_doorbell_control = in gfx_v7_0_mqd_init()
2840 if (ring->use_doorbell) in gfx_v7_0_mqd_init()
2841 mqd->cp_hqd_pq_doorbell_control |= CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK; in gfx_v7_0_mqd_init()
2843 mqd->cp_hqd_pq_doorbell_control &= ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK; in gfx_v7_0_mqd_init()
2846 mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc; in gfx_v7_0_mqd_init()
2847 mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); in gfx_v7_0_mqd_init()
2850 mqd->cp_mqd_control = RREG32(mmCP_MQD_CONTROL); in gfx_v7_0_mqd_init()
2851 mqd->cp_mqd_control &= ~CP_MQD_CONTROL__VMID_MASK; in gfx_v7_0_mqd_init()
2854 hqd_gpu_addr = ring->gpu_addr >> 8; in gfx_v7_0_mqd_init()
2855 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; in gfx_v7_0_mqd_init()
2856 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); in gfx_v7_0_mqd_init()
2859 mqd->cp_hqd_pq_control = RREG32(mmCP_HQD_PQ_CONTROL); in gfx_v7_0_mqd_init()
2860 mqd->cp_hqd_pq_control &= in gfx_v7_0_mqd_init()
2864 mqd->cp_hqd_pq_control |= in gfx_v7_0_mqd_init()
2865 order_base_2(ring->ring_size / 8); in gfx_v7_0_mqd_init()
2866 mqd->cp_hqd_pq_control |= in gfx_v7_0_mqd_init()
2869 mqd->cp_hqd_pq_control |= in gfx_v7_0_mqd_init()
2872 mqd->cp_hqd_pq_control &= in gfx_v7_0_mqd_init()
2876 mqd->cp_hqd_pq_control |= in gfx_v7_0_mqd_init()
2881 wb_gpu_addr = ring->wptr_gpu_addr; in gfx_v7_0_mqd_init()
2882 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v7_0_mqd_init()
2883 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v7_0_mqd_init()
2886 wb_gpu_addr = ring->rptr_gpu_addr; in gfx_v7_0_mqd_init()
2887 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v7_0_mqd_init()
2888 mqd->cp_hqd_pq_rptr_report_addr_hi = in gfx_v7_0_mqd_init()
2892 if (ring->use_doorbell) { in gfx_v7_0_mqd_init()
2893 mqd->cp_hqd_pq_doorbell_control = in gfx_v7_0_mqd_init()
2895 mqd->cp_hqd_pq_doorbell_control &= in gfx_v7_0_mqd_init()
2897 mqd->cp_hqd_pq_doorbell_control |= in gfx_v7_0_mqd_init()
2898 (ring->doorbell_index << in gfx_v7_0_mqd_init()
2900 mqd->cp_hqd_pq_doorbell_control |= in gfx_v7_0_mqd_init()
2902 mqd->cp_hqd_pq_doorbell_control &= in gfx_v7_0_mqd_init()
2907 mqd->cp_hqd_pq_doorbell_control = 0; in gfx_v7_0_mqd_init()
2911 ring->wptr = 0; in gfx_v7_0_mqd_init()
2912 mqd->cp_hqd_pq_wptr = lower_32_bits(ring->wptr); in gfx_v7_0_mqd_init()
2913 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); in gfx_v7_0_mqd_init()
2916 mqd->cp_hqd_vmid = 0; in gfx_v7_0_mqd_init()
2919 mqd->cp_hqd_ib_control = RREG32(mmCP_HQD_IB_CONTROL); in gfx_v7_0_mqd_init()
2920 mqd->cp_hqd_ib_base_addr_lo = RREG32(mmCP_HQD_IB_BASE_ADDR); in gfx_v7_0_mqd_init()
2921 mqd->cp_hqd_ib_base_addr_hi = RREG32(mmCP_HQD_IB_BASE_ADDR_HI); in gfx_v7_0_mqd_init()
2922 mqd->cp_hqd_ib_rptr = RREG32(mmCP_HQD_IB_RPTR); in gfx_v7_0_mqd_init()
2923 mqd->cp_hqd_persistent_state = RREG32(mmCP_HQD_PERSISTENT_STATE); in gfx_v7_0_mqd_init()
2924 mqd->cp_hqd_sema_cmd = RREG32(mmCP_HQD_SEMA_CMD); in gfx_v7_0_mqd_init()
2925 mqd->cp_hqd_msg_type = RREG32(mmCP_HQD_MSG_TYPE); in gfx_v7_0_mqd_init()
2926 mqd->cp_hqd_atomic0_preop_lo = RREG32(mmCP_HQD_ATOMIC0_PREOP_LO); in gfx_v7_0_mqd_init()
2927 mqd->cp_hqd_atomic0_preop_hi = RREG32(mmCP_HQD_ATOMIC0_PREOP_HI); in gfx_v7_0_mqd_init()
2928 mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO); in gfx_v7_0_mqd_init()
2929 mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI); in gfx_v7_0_mqd_init()
2930 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); in gfx_v7_0_mqd_init()
2931 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); in gfx_v7_0_mqd_init()
2932 mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY); in gfx_v7_0_mqd_init()
2933 mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY); in gfx_v7_0_mqd_init()
2934 mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR); in gfx_v7_0_mqd_init()
2937 mqd->cp_hqd_active = 1; in gfx_v7_0_mqd_init()
2947 mqd_data = &mqd->cp_mqd_base_addr_lo; in gfx_v7_0_mqd_commit()
2949 /* disable wptr polling */ in gfx_v7_0_mqd_commit()
2956 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); in gfx_v7_0_mqd_commit()
2960 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); in gfx_v7_0_mqd_commit()
2970 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; in gfx_v7_0_compute_queue_init()
2973 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, in gfx_v7_0_compute_queue_init()
2976 dev_warn(adev->dev, "(%d) create MQD bo failed\n", r); in gfx_v7_0_compute_queue_init()
2980 mutex_lock(&adev->srbm_mutex); in gfx_v7_0_compute_queue_init()
2981 cik_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v7_0_compute_queue_init()
2988 mutex_unlock(&adev->srbm_mutex); in gfx_v7_0_compute_queue_init()
2990 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v7_0_compute_queue_init()
2991 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v7_0_compute_queue_init()
2996 * gfx_v7_0_cp_compute_resume - setup the compute queue registers
3016 for (i = 0; i < adev->gfx.mec.num_mec; i++) in gfx_v7_0_cp_compute_resume()
3017 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) in gfx_v7_0_cp_compute_resume()
3021 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v7_0_cp_compute_resume()
3031 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v7_0_cp_compute_resume()
3032 ring = &adev->gfx.compute_ring[i]; in gfx_v7_0_cp_compute_resume()
3096 * gfx_v7_0_ring_emit_pipeline_sync - cik vm flush using the CP
3105 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v7_0_ring_emit_pipeline_sync()
3106 uint32_t seq = ring->fence_drv.sync_seq; in gfx_v7_0_ring_emit_pipeline_sync()
3107 uint64_t addr = ring->fence_drv.gpu_addr; in gfx_v7_0_ring_emit_pipeline_sync()
3131 * VMIDs 1-15 are used for userspace clients and are handled
3135 * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP
3147 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v7_0_ring_emit_vm_flush()
3179 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v7_0_ring_emit_wreg()
3191 * The RLC is a multi-purpose microengine that handles a
3202 if (adev->flags & AMD_IS_APU) { in gfx_v7_0_rlc_init()
3203 if (adev->asic_type == CHIP_KAVERI) { in gfx_v7_0_rlc_init()
3204 adev->gfx.rlc.reg_list = spectre_rlc_save_restore_register_list; in gfx_v7_0_rlc_init()
3205 adev->gfx.rlc.reg_list_size = in gfx_v7_0_rlc_init()
3208 adev->gfx.rlc.reg_list = kalindi_rlc_save_restore_register_list; in gfx_v7_0_rlc_init()
3209 adev->gfx.rlc.reg_list_size = in gfx_v7_0_rlc_init()
3213 adev->gfx.rlc.cs_data = ci_cs_data; in gfx_v7_0_rlc_init()
3214 adev->gfx.rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */ in gfx_v7_0_rlc_init()
3215 adev->gfx.rlc.cp_table_size += 64 * 1024; /* GDS */ in gfx_v7_0_rlc_init()
3217 src_ptr = adev->gfx.rlc.reg_list; in gfx_v7_0_rlc_init()
3218 dws = adev->gfx.rlc.reg_list_size; in gfx_v7_0_rlc_init()
3221 cs_data = adev->gfx.rlc.cs_data; in gfx_v7_0_rlc_init()
3237 if (adev->gfx.rlc.cp_table_size) { in gfx_v7_0_rlc_init()
3244 if (adev->gfx.rlc.funcs->update_spm_vmid) in gfx_v7_0_rlc_init()
3245 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf); in gfx_v7_0_rlc_init()
3267 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_wait_for_rlc_serdes()
3268 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v7_0_wait_for_rlc_serdes()
3269 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v7_0_wait_for_rlc_serdes()
3271 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v7_0_wait_for_rlc_serdes()
3279 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_wait_for_rlc_serdes()
3285 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v7_0_wait_for_rlc_serdes()
3313 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_halt_rlc()
3339 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_set_safe_mode()
3345 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_set_safe_mode()
3361 * gfx_v7_0_rlc_stop - stop the RLC ME
3377 * gfx_v7_0_rlc_start - start the RLC ME
3405 * gfx_v7_0_rlc_resume - setup the RLC hw
3411 * Returns 0 for success, -EINVAL if the ucode is not available.
3420 if (!adev->gfx.rlc_fw) in gfx_v7_0_rlc_resume()
3421 return -EINVAL; in gfx_v7_0_rlc_resume()
3423 hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data; in gfx_v7_0_rlc_resume()
3424 amdgpu_ucode_print_rlc_hdr(&hdr->header); in gfx_v7_0_rlc_resume()
3425 adev->gfx.rlc_fw_version = le32_to_cpu(hdr->header.ucode_version); in gfx_v7_0_rlc_resume()
3426 adev->gfx.rlc_feature_version = le32_to_cpu( in gfx_v7_0_rlc_resume()
3427 hdr->ucode_feature_version); in gfx_v7_0_rlc_resume()
3429 adev->gfx.rlc.funcs->stop(adev); in gfx_v7_0_rlc_resume()
3431 /* disable CG */ in gfx_v7_0_rlc_resume()
3435 adev->gfx.rlc.funcs->reset(adev); in gfx_v7_0_rlc_resume()
3442 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_rlc_resume()
3447 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_rlc_resume()
3453 (adev->gfx.rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in gfx_v7_0_rlc_resume()
3454 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; in gfx_v7_0_rlc_resume()
3458 WREG32(mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v7_0_rlc_resume()
3460 /* XXX - find out what chips support lbpw */ in gfx_v7_0_rlc_resume()
3463 if (adev->asic_type == CHIP_BONAIRE) in gfx_v7_0_rlc_resume()
3466 adev->gfx.rlc.funcs->start(adev); in gfx_v7_0_rlc_resume()
3493 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { in gfx_v7_0_enable_cgcg()
3498 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_cgcg()
3506 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_cgcg()
3534 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { in gfx_v7_0_enable_mgcg()
3535 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { in gfx_v7_0_enable_mgcg()
3536 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { in gfx_v7_0_enable_mgcg()
3552 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_mgcg()
3559 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_mgcg()
3563 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { in gfx_v7_0_enable_mgcg()
3569 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && in gfx_v7_0_enable_mgcg()
3570 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) in gfx_v7_0_enable_mgcg()
3603 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_mgcg()
3609 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_enable_mgcg()
3636 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS)) in gfx_v7_0_enable_sclk_slowdown_on_pu()
3650 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS)) in gfx_v7_0_enable_sclk_slowdown_on_pd()
3663 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP)) in gfx_v7_0_enable_cp_pg()
3676 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GDS)) in gfx_v7_0_enable_gds_pg()
3686 if (adev->asic_type == CHIP_KAVERI) in gfx_v7_0_cp_pg_table_num()
3697 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { in gfx_v7_0_enable_gfx_cgpg()
3746 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); in gfx_v7_0_get_cu_active_bitmap()
3755 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); in gfx_v7_0_init_ao_cu_mask()
3759 tmp |= (adev->gfx.cu_info.number << RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT); in gfx_v7_0_init_ao_cu_mask()
3769 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG)) in gfx_v7_0_enable_gfx_static_mgpg()
3783 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG)) in gfx_v7_0_enable_gfx_dynamic_mgpg()
3799 if (adev->gfx.rlc.cs_data) { in gfx_v7_0_init_gfx_cgpg()
3801 WREG32(mmRLC_GPM_SCRATCH_DATA, upper_32_bits(adev->gfx.rlc.clear_state_gpu_addr)); in gfx_v7_0_init_gfx_cgpg()
3802 WREG32(mmRLC_GPM_SCRATCH_DATA, lower_32_bits(adev->gfx.rlc.clear_state_gpu_addr)); in gfx_v7_0_init_gfx_cgpg()
3803 WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.clear_state_size); in gfx_v7_0_init_gfx_cgpg()
3809 if (adev->gfx.rlc.reg_list) { in gfx_v7_0_init_gfx_cgpg()
3811 for (i = 0; i < adev->gfx.rlc.reg_list_size; i++) in gfx_v7_0_init_gfx_cgpg()
3812 WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.reg_list[i]); in gfx_v7_0_init_gfx_cgpg()
3820 WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8); in gfx_v7_0_init_gfx_cgpg()
3821 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); in gfx_v7_0_init_gfx_cgpg()
3856 if (adev->gfx.rlc.cs_data == NULL) in gfx_v7_0_get_csb_size()
3864 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v7_0_get_csb_size()
3865 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v7_0_get_csb_size()
3866 if (sect->id == SECT_CONTEXT) in gfx_v7_0_get_csb_size()
3867 count += 2 + ext->reg_count; in gfx_v7_0_get_csb_size()
3889 if (adev->gfx.rlc.cs_data == NULL) in gfx_v7_0_get_csb_buffer()
3901 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v7_0_get_csb_buffer()
3902 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v7_0_get_csb_buffer()
3903 if (sect->id == SECT_CONTEXT) { in gfx_v7_0_get_csb_buffer()
3905 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); in gfx_v7_0_get_csb_buffer()
3906 buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START); in gfx_v7_0_get_csb_buffer()
3907 for (i = 0; i < ext->reg_count; i++) in gfx_v7_0_get_csb_buffer()
3908 buffer[count++] = cpu_to_le32(ext->extent[i]); in gfx_v7_0_get_csb_buffer()
3916 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); in gfx_v7_0_get_csb_buffer()
3917 switch (adev->asic_type) { in gfx_v7_0_get_csb_buffer()
3950 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | in gfx_v7_0_init_pg()
3958 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) { in gfx_v7_0_init_pg()
3970 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | in gfx_v7_0_fini_pg()
3977 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) { in gfx_v7_0_fini_pg()
3985 * gfx_v7_0_get_gpu_clock_counter - return GPU clock counter snapshot
3996 mutex_lock(&adev->gfx.gpu_clock_mutex); in gfx_v7_0_get_gpu_clock_counter()
4000 mutex_unlock(&adev->gfx.gpu_clock_mutex); in gfx_v7_0_get_gpu_clock_counter()
4040 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); in gfx_v7_0_ring_emit_gds_switch()
4045 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_ring_soft_recovery()
4076 while (num--) in wave_read_regs()
4145 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_early_init()
4147 adev->gfx.xcc_mask = 1; in gfx_v7_0_early_init()
4148 adev->gfx.num_gfx_rings = GFX7_NUM_GFX_RINGS; in gfx_v7_0_early_init()
4149 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), in gfx_v7_0_early_init()
4151 adev->gfx.funcs = &gfx_v7_0_gfx_funcs; in gfx_v7_0_early_init()
4152 adev->gfx.rlc.funcs = &gfx_v7_0_rlc_funcs; in gfx_v7_0_early_init()
4162 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_late_init()
4165 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v7_0_late_init()
4169 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v7_0_late_init()
4183 switch (adev->asic_type) { in gfx_v7_0_gpu_early_init()
4185 adev->gfx.config.max_shader_engines = 2; in gfx_v7_0_gpu_early_init()
4186 adev->gfx.config.max_tile_pipes = 4; in gfx_v7_0_gpu_early_init()
4187 adev->gfx.config.max_cu_per_sh = 7; in gfx_v7_0_gpu_early_init()
4188 adev->gfx.config.max_sh_per_se = 1; in gfx_v7_0_gpu_early_init()
4189 adev->gfx.config.max_backends_per_se = 2; in gfx_v7_0_gpu_early_init()
4190 adev->gfx.config.max_texture_channel_caches = 4; in gfx_v7_0_gpu_early_init()
4191 adev->gfx.config.max_gprs = 256; in gfx_v7_0_gpu_early_init()
4192 adev->gfx.config.max_gs_threads = 32; in gfx_v7_0_gpu_early_init()
4193 adev->gfx.config.max_hw_contexts = 8; in gfx_v7_0_gpu_early_init()
4195 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v7_0_gpu_early_init()
4196 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v7_0_gpu_early_init()
4197 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v7_0_gpu_early_init()
4198 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v7_0_gpu_early_init()
4202 adev->gfx.config.max_shader_engines = 4; in gfx_v7_0_gpu_early_init()
4203 adev->gfx.config.max_tile_pipes = 16; in gfx_v7_0_gpu_early_init()
4204 adev->gfx.config.max_cu_per_sh = 11; in gfx_v7_0_gpu_early_init()
4205 adev->gfx.config.max_sh_per_se = 1; in gfx_v7_0_gpu_early_init()
4206 adev->gfx.config.max_backends_per_se = 4; in gfx_v7_0_gpu_early_init()
4207 adev->gfx.config.max_texture_channel_caches = 16; in gfx_v7_0_gpu_early_init()
4208 adev->gfx.config.max_gprs = 256; in gfx_v7_0_gpu_early_init()
4209 adev->gfx.config.max_gs_threads = 32; in gfx_v7_0_gpu_early_init()
4210 adev->gfx.config.max_hw_contexts = 8; in gfx_v7_0_gpu_early_init()
4212 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v7_0_gpu_early_init()
4213 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v7_0_gpu_early_init()
4214 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v7_0_gpu_early_init()
4215 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v7_0_gpu_early_init()
4219 adev->gfx.config.max_shader_engines = 1; in gfx_v7_0_gpu_early_init()
4220 adev->gfx.config.max_tile_pipes = 4; in gfx_v7_0_gpu_early_init()
4221 adev->gfx.config.max_cu_per_sh = 8; in gfx_v7_0_gpu_early_init()
4222 adev->gfx.config.max_backends_per_se = 2; in gfx_v7_0_gpu_early_init()
4223 adev->gfx.config.max_sh_per_se = 1; in gfx_v7_0_gpu_early_init()
4224 adev->gfx.config.max_texture_channel_caches = 4; in gfx_v7_0_gpu_early_init()
4225 adev->gfx.config.max_gprs = 256; in gfx_v7_0_gpu_early_init()
4226 adev->gfx.config.max_gs_threads = 16; in gfx_v7_0_gpu_early_init()
4227 adev->gfx.config.max_hw_contexts = 8; in gfx_v7_0_gpu_early_init()
4229 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v7_0_gpu_early_init()
4230 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v7_0_gpu_early_init()
4231 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v7_0_gpu_early_init()
4232 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v7_0_gpu_early_init()
4238 adev->gfx.config.max_shader_engines = 1; in gfx_v7_0_gpu_early_init()
4239 adev->gfx.config.max_tile_pipes = 2; in gfx_v7_0_gpu_early_init()
4240 adev->gfx.config.max_cu_per_sh = 2; in gfx_v7_0_gpu_early_init()
4241 adev->gfx.config.max_sh_per_se = 1; in gfx_v7_0_gpu_early_init()
4242 adev->gfx.config.max_backends_per_se = 1; in gfx_v7_0_gpu_early_init()
4243 adev->gfx.config.max_texture_channel_caches = 2; in gfx_v7_0_gpu_early_init()
4244 adev->gfx.config.max_gprs = 256; in gfx_v7_0_gpu_early_init()
4245 adev->gfx.config.max_gs_threads = 16; in gfx_v7_0_gpu_early_init()
4246 adev->gfx.config.max_hw_contexts = 8; in gfx_v7_0_gpu_early_init()
4248 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v7_0_gpu_early_init()
4249 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v7_0_gpu_early_init()
4250 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v7_0_gpu_early_init()
4251 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v7_0_gpu_early_init()
4256 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); in gfx_v7_0_gpu_early_init()
4257 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; in gfx_v7_0_gpu_early_init()
4259 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, in gfx_v7_0_gpu_early_init()
4261 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, in gfx_v7_0_gpu_early_init()
4264 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; in gfx_v7_0_gpu_early_init()
4265 adev->gfx.config.mem_max_burst_length_bytes = 256; in gfx_v7_0_gpu_early_init()
4266 if (adev->flags & AMD_IS_APU) { in gfx_v7_0_gpu_early_init()
4289 adev->gfx.config.mem_row_size_in_kb = 2; in gfx_v7_0_gpu_early_init()
4291 adev->gfx.config.mem_row_size_in_kb = 1; in gfx_v7_0_gpu_early_init()
4294 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; in gfx_v7_0_gpu_early_init()
4295 if (adev->gfx.config.mem_row_size_in_kb > 4) in gfx_v7_0_gpu_early_init()
4296 adev->gfx.config.mem_row_size_in_kb = 4; in gfx_v7_0_gpu_early_init()
4299 adev->gfx.config.shader_engine_tile_size = 32; in gfx_v7_0_gpu_early_init()
4300 adev->gfx.config.num_gpus = 1; in gfx_v7_0_gpu_early_init()
4301 adev->gfx.config.multi_gpu_tile_size = 64; in gfx_v7_0_gpu_early_init()
4305 switch (adev->gfx.config.mem_row_size_in_kb) { in gfx_v7_0_gpu_early_init()
4317 adev->gfx.config.gb_addr_config = gb_addr_config; in gfx_v7_0_gpu_early_init()
4325 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; in gfx_v7_0_compute_ring_init()
4328 ring->me = mec + 1; in gfx_v7_0_compute_ring_init()
4329 ring->pipe = pipe; in gfx_v7_0_compute_ring_init()
4330 ring->queue = queue; in gfx_v7_0_compute_ring_init()
4332 ring->ring_obj = NULL; in gfx_v7_0_compute_ring_init()
4333 ring->use_doorbell = true; in gfx_v7_0_compute_ring_init()
4334 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; in gfx_v7_0_compute_ring_init()
4335 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); in gfx_v7_0_compute_ring_init()
4338 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) in gfx_v7_0_compute_ring_init()
4339 + ring->pipe; in gfx_v7_0_compute_ring_init()
4341 /* type-2 packets are deprecated on MEC, use type-3 instead */ in gfx_v7_0_compute_ring_init()
4343 &adev->gfx.eop_irq, irq_type, in gfx_v7_0_compute_ring_init()
4355 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_sw_init()
4358 switch (adev->asic_type) { in gfx_v7_0_sw_init()
4360 adev->gfx.mec.num_mec = 2; in gfx_v7_0_sw_init()
4367 adev->gfx.mec.num_mec = 1; in gfx_v7_0_sw_init()
4370 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v7_0_sw_init()
4371 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v7_0_sw_init()
4373 /* EOP Event */ in gfx_v7_0_sw_init()
4374 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 181, &adev->gfx.eop_irq); in gfx_v7_0_sw_init()
4380 &adev->gfx.priv_reg_irq); in gfx_v7_0_sw_init()
4386 &adev->gfx.priv_inst_irq); in gfx_v7_0_sw_init()
4396 r = adev->gfx.rlc.funcs->init(adev); in gfx_v7_0_sw_init()
4409 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v7_0_sw_init()
4410 ring = &adev->gfx.gfx_ring[i]; in gfx_v7_0_sw_init()
4411 ring->ring_obj = NULL; in gfx_v7_0_sw_init()
4412 sprintf(ring->name, "gfx"); in gfx_v7_0_sw_init()
4414 &adev->gfx.eop_irq, in gfx_v7_0_sw_init()
4421 /* set up the compute queues - allocate horizontally across pipes */ in gfx_v7_0_sw_init()
4423 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v7_0_sw_init()
4424 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v7_0_sw_init()
4425 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v7_0_sw_init()
4441 adev->gfx.ce_ram_size = 0x8000; in gfx_v7_0_sw_init()
4450 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_sw_fini()
4453 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v7_0_sw_fini()
4454 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); in gfx_v7_0_sw_fini()
4455 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v7_0_sw_fini()
4456 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); in gfx_v7_0_sw_fini()
4461 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, in gfx_v7_0_sw_fini()
4462 &adev->gfx.rlc.clear_state_gpu_addr, in gfx_v7_0_sw_fini()
4463 (void **)&adev->gfx.rlc.cs_ptr); in gfx_v7_0_sw_fini()
4464 if (adev->gfx.rlc.cp_table_size) { in gfx_v7_0_sw_fini()
4465 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, in gfx_v7_0_sw_fini()
4466 &adev->gfx.rlc.cp_table_gpu_addr, in gfx_v7_0_sw_fini()
4467 (void **)&adev->gfx.rlc.cp_table_ptr); in gfx_v7_0_sw_fini()
4477 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_hw_init()
4482 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); in gfx_v7_0_hw_init()
4484 r = adev->gfx.rlc.funcs->resume(adev); in gfx_v7_0_hw_init()
4497 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_hw_fini()
4499 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v7_0_hw_fini()
4500 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v7_0_hw_fini()
4502 adev->gfx.rlc.funcs->stop(adev); in gfx_v7_0_hw_fini()
4520 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_is_idle()
4532 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_wait_for_idle()
4534 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v7_0_wait_for_idle()
4542 return -ETIMEDOUT; in gfx_v7_0_wait_for_idle()
4549 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_soft_reset()
4578 /* disable CG/PG */ in gfx_v7_0_soft_reset()
4583 adev->gfx.rlc.funcs->stop(adev); in gfx_v7_0_soft_reset()
4585 /* Disable GFX parsing/prefetching */ in gfx_v7_0_soft_reset()
4588 /* Disable MEC parsing/prefetching */ in gfx_v7_0_soft_reset()
4594 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in gfx_v7_0_soft_reset()
4608 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); in gfx_v7_0_soft_reset()
4793 DRM_DEBUG("IH: CP EOP\n"); in gfx_v7_0_eop_irq()
4794 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v7_0_eop_irq()
4795 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v7_0_eop_irq()
4798 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); in gfx_v7_0_eop_irq()
4802 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v7_0_eop_irq()
4803 ring = &adev->gfx.compute_ring[i]; in gfx_v7_0_eop_irq()
4804 if ((ring->me == me_id) && (ring->pipe == pipe_id)) in gfx_v7_0_eop_irq()
4819 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v7_0_fault()
4820 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v7_0_fault()
4823 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); in gfx_v7_0_fault()
4827 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v7_0_fault()
4828 ring = &adev->gfx.compute_ring[i]; in gfx_v7_0_fault()
4829 if ((ring->me == me_id) && (ring->pipe == pipe_id)) in gfx_v7_0_fault()
4830 drm_sched_fault(&ring->sched); in gfx_v7_0_fault()
4859 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_set_clockgating_state()
4882 struct amdgpu_device *adev = ip_block->adev; in gfx_v7_0_set_powergating_state()
4887 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | in gfx_v7_0_set_powergating_state()
4894 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) { in gfx_v7_0_set_powergating_state()
4959 struct amdgpu_device *adev = ring->adev; in gfx_v7_0_reset_kgq()
4960 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v7_0_reset_kgq()
4961 struct amdgpu_ring *kiq_ring = &kiq->ring; in gfx_v7_0_reset_kgq()
4967 return -EINVAL; in gfx_v7_0_reset_kgq()
4969 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) in gfx_v7_0_reset_kgq()
4970 return -EINVAL; in gfx_v7_0_reset_kgq()
4972 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v7_0_reset_kgq()
4975 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v7_0_reset_kgq()
4976 return -ENOMEM; in gfx_v7_0_reset_kgq()
4983 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v7_0_reset_kgq()
4990 return -ENOMEM; in gfx_v7_0_reset_kgq()
4991 gfx_v7_0_ring_emit_fence_gfx(ring, ring->fence_drv.gpu_addr, in gfx_v7_0_reset_kgq()
4992 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC); in gfx_v7_0_reset_kgq()
5087 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v7_0_set_ring_funcs()
5088 adev->gfx.gfx_ring[i].funcs = &gfx_v7_0_ring_funcs_gfx; in gfx_v7_0_set_ring_funcs()
5089 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v7_0_set_ring_funcs()
5090 adev->gfx.compute_ring[i].funcs = &gfx_v7_0_ring_funcs_compute; in gfx_v7_0_set_ring_funcs()
5110 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; in gfx_v7_0_set_irq_funcs()
5111 adev->gfx.eop_irq.funcs = &gfx_v7_0_eop_irq_funcs; in gfx_v7_0_set_irq_funcs()
5113 adev->gfx.priv_reg_irq.num_types = 1; in gfx_v7_0_set_irq_funcs()
5114 adev->gfx.priv_reg_irq.funcs = &gfx_v7_0_priv_reg_irq_funcs; in gfx_v7_0_set_irq_funcs()
5116 adev->gfx.priv_inst_irq.num_types = 1; in gfx_v7_0_set_irq_funcs()
5117 adev->gfx.priv_inst_irq.funcs = &gfx_v7_0_priv_inst_irq_funcs; in gfx_v7_0_set_irq_funcs()
5123 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); in gfx_v7_0_set_gds_init()
5124 adev->gds.gws_size = 64; in gfx_v7_0_set_gds_init()
5125 adev->gds.oa_size = 16; in gfx_v7_0_set_gds_init()
5126 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); in gfx_v7_0_set_gds_init()
5134 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; in gfx_v7_0_get_cu_info()
5138 if (adev->flags & AMD_IS_APU) in gfx_v7_0_get_cu_info()
5141 ao_cu_num = adev->gfx.config.max_cu_per_sh; in gfx_v7_0_get_cu_info()
5147 mutex_lock(&adev->grbm_idx_mutex); in gfx_v7_0_get_cu_info()
5148 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v7_0_get_cu_info()
5149 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v7_0_get_cu_info()
5158 cu_info->bitmap[0][i][j] = bitmap; in gfx_v7_0_get_cu_info()
5160 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { in gfx_v7_0_get_cu_info()
5171 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; in gfx_v7_0_get_cu_info()
5175 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v7_0_get_cu_info()
5177 cu_info->number = active_cu_number; in gfx_v7_0_get_cu_info()
5178 cu_info->ao_cu_mask = ao_cu_mask; in gfx_v7_0_get_cu_info()
5179 cu_info->simd_per_cu = NUM_SIMD_PER_CU; in gfx_v7_0_get_cu_info()
5180 cu_info->max_waves_per_simd = 10; in gfx_v7_0_get_cu_info()
5181 cu_info->max_scratch_slots_per_cu = 32; in gfx_v7_0_get_cu_info()
5182 cu_info->wave_front_size = 64; in gfx_v7_0_get_cu_info()
5183 cu_info->lds_size = 64; in gfx_v7_0_get_cu_info()