1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "mmhub_v1_8.h" 25 26 #include "mmhub/mmhub_1_8_0_offset.h" 27 #include "mmhub/mmhub_1_8_0_sh_mask.h" 28 #include "vega10_enum.h" 29 30 #include "soc15_common.h" 31 #include "soc15.h" 32 #include "amdgpu_ras.h" 33 #include "amdgpu_psp.h" 34 35 #define regVM_L2_CNTL3_DEFAULT 0x80100007 36 #define regVM_L2_CNTL4_DEFAULT 0x000000c1 37 38 static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev) 39 { 40 u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE); 41 u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP); 42 43 base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 44 base <<= 24; 45 46 top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; 47 top <<= 24; 48 49 adev->gmc.fb_start = base; 50 adev->gmc.fb_end = top; 51 52 return base; 53 } 54 55 static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 56 uint64_t page_table_base) 57 { 58 struct amdgpu_vmhub *hub; 59 u32 inst_mask; 60 int i; 61 62 inst_mask = adev->aid_mask; 63 for_each_inst(i, inst_mask) { 64 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 65 WREG32_SOC15_OFFSET(MMHUB, i, 66 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 67 hub->ctx_addr_distance * vmid, 68 lower_32_bits(page_table_base)); 69 70 WREG32_SOC15_OFFSET(MMHUB, i, 71 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 72 hub->ctx_addr_distance * vmid, 73 upper_32_bits(page_table_base)); 74 } 75 } 76 77 static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev) 78 { 79 uint64_t pt_base; 80 u32 inst_mask; 81 int i; 82 83 if (adev->gmc.pdb0_bo) 84 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 85 else 86 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 87 88 mmhub_v1_8_setup_vm_pt_regs(adev, 0, pt_base); 89 90 /* If use GART for FB translation, vmid0 page table covers both 91 * vram and system memory (gart) 92 */ 93 inst_mask = adev->aid_mask; 94 for_each_inst(i, inst_mask) { 95 if (adev->gmc.pdb0_bo) { 96 WREG32_SOC15(MMHUB, i, 97 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 98 (u32)(adev->gmc.fb_start >> 12)); 99 WREG32_SOC15(MMHUB, i, 100 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 101 (u32)(adev->gmc.fb_start >> 44)); 102 103 WREG32_SOC15(MMHUB, i, 104 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 105 (u32)(adev->gmc.gart_end >> 12)); 106 WREG32_SOC15(MMHUB, i, 107 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 108 (u32)(adev->gmc.gart_end >> 44)); 109 110 } else { 111 WREG32_SOC15(MMHUB, i, 112 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 113 (u32)(adev->gmc.gart_start >> 12)); 114 WREG32_SOC15(MMHUB, i, 115 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 116 (u32)(adev->gmc.gart_start >> 44)); 117 118 WREG32_SOC15(MMHUB, i, 119 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 120 (u32)(adev->gmc.gart_end >> 12)); 121 WREG32_SOC15(MMHUB, i, 122 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 123 (u32)(adev->gmc.gart_end >> 44)); 124 } 125 } 126 } 127 128 static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev) 129 { 130 uint32_t tmp, inst_mask; 131 uint64_t value; 132 int i; 133 134 if (amdgpu_sriov_vf(adev)) 135 return; 136 137 inst_mask = adev->aid_mask; 138 for_each_inst(i, inst_mask) { 139 /* Program the AGP BAR */ 140 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BASE, 0); 141 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 142 adev->gmc.agp_start >> 24); 143 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 144 adev->gmc.agp_end >> 24); 145 146 /* Program the system aperture low logical page number. */ 147 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 148 min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 149 150 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 151 max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 152 153 /* In the case squeezing vram into GART aperture, we don't use 154 * FB aperture and AGP aperture. Disable them. 155 */ 156 if (adev->gmc.pdb0_bo) { 157 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 0xFFFFFF); 158 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 0); 159 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_TOP, 0); 160 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_BASE, 161 0x00FFFFFF); 162 WREG32_SOC15(MMHUB, i, 163 regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 164 0x3FFFFFFF); 165 WREG32_SOC15(MMHUB, i, 166 regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0); 167 } 168 169 /* Set default page address. */ 170 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 171 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 172 (u32)(value >> 12)); 173 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 174 (u32)(value >> 44)); 175 176 /* Program "protection fault". */ 177 WREG32_SOC15(MMHUB, i, 178 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 179 (u32)(adev->dummy_page_addr >> 12)); 180 WREG32_SOC15(MMHUB, i, 181 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 182 (u32)((u64)adev->dummy_page_addr >> 44)); 183 184 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2); 185 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, 186 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 187 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2, tmp); 188 } 189 } 190 191 static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) 192 { 193 uint32_t tmp, inst_mask; 194 int i; 195 196 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 197 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 198 199 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 200 1); 201 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 202 SYSTEM_ACCESS_MODE, 3); 203 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 204 ENABLE_ADVANCED_DRIVER_MODEL, 1); 205 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 206 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 207 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 208 MTYPE, MTYPE_UC);/* XXX for emulation. */ 209 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 210 211 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 212 } else { 213 inst_mask = adev->aid_mask; 214 for_each_inst(i, inst_mask) { 215 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 216 217 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 218 1); 219 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 220 SYSTEM_ACCESS_MODE, 3); 221 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 222 ENABLE_ADVANCED_DRIVER_MODEL, 1); 223 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 224 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 225 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 226 MTYPE, MTYPE_UC);/* XXX for emulation. */ 227 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 228 229 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 230 } 231 } 232 } 233 234 /* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ 235 static void mmhub_v1_8_init_snoop_override_regs(struct amdgpu_device *adev) 236 { 237 uint32_t tmp, inst_mask; 238 int i, j; 239 uint32_t distance = regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - 240 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; 241 242 if (amdgpu_sriov_vf(adev)) 243 return; 244 245 inst_mask = adev->aid_mask; 246 for_each_inst(i, inst_mask) { 247 for (j = 0; j < 5; j++) { /* DAGB instances */ 248 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 249 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance); 250 tmp |= (1 << 15); /* SDMA client is BIT15 */ 251 WREG32_SOC15_OFFSET(MMHUB, i, 252 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance, tmp); 253 254 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 255 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance); 256 tmp |= (1 << 15); 257 WREG32_SOC15_OFFSET(MMHUB, i, 258 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance, tmp); 259 } 260 } 261 } 262 263 static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) 264 { 265 uint32_t tmp, inst_mask; 266 int i; 267 268 if (amdgpu_sriov_vf(adev)) 269 return; 270 271 /* Setup L2 cache */ 272 inst_mask = adev->aid_mask; 273 for_each_inst(i, inst_mask) { 274 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL); 275 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); 276 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 277 ENABLE_L2_FRAGMENT_PROCESSING, 1); 278 /* XXX for emulation, Refer to closed source code.*/ 279 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 280 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 281 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 282 0); 283 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 284 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 285 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 286 IDENTITY_MODE_FRAGMENT_SIZE, 0); 287 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL, tmp); 288 289 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL2); 290 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 291 1); 292 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 293 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL2, tmp); 294 295 tmp = regVM_L2_CNTL3_DEFAULT; 296 if (adev->gmc.translate_further) { 297 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); 298 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 299 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 300 } else { 301 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); 302 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 303 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 304 } 305 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL3, tmp); 306 307 tmp = regVM_L2_CNTL4_DEFAULT; 308 /* For AMD APP APUs setup WC memory */ 309 if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) { 310 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 311 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 312 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 313 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 314 } else { 315 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 316 VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 317 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 318 VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 319 } 320 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL4, tmp); 321 } 322 } 323 324 static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev) 325 { 326 uint32_t tmp, inst_mask; 327 int i; 328 329 inst_mask = adev->aid_mask; 330 for_each_inst(i, inst_mask) { 331 tmp = RREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL); 332 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 333 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 334 adev->gmc.vmid0_page_table_depth); 335 tmp = REG_SET_FIELD(tmp, 336 VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE, 337 adev->gmc.vmid0_page_table_block_size); 338 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, 339 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 340 WREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL, tmp); 341 } 342 } 343 344 static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev) 345 { 346 u32 inst_mask; 347 int i; 348 349 if (amdgpu_sriov_vf(adev)) 350 return; 351 352 inst_mask = adev->aid_mask; 353 for_each_inst(i, inst_mask) { 354 WREG32_SOC15(MMHUB, i, 355 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 356 0XFFFFFFFF); 357 WREG32_SOC15(MMHUB, i, 358 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 359 0x0000000F); 360 361 WREG32_SOC15(MMHUB, i, 362 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 363 0); 364 WREG32_SOC15(MMHUB, i, 365 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 366 0); 367 368 WREG32_SOC15(MMHUB, i, 369 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 370 WREG32_SOC15(MMHUB, i, 371 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 372 } 373 } 374 375 static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev) 376 { 377 struct amdgpu_vmhub *hub; 378 unsigned int num_level, block_size; 379 uint32_t tmp, inst_mask; 380 int i, j; 381 382 num_level = adev->vm_manager.num_level; 383 block_size = adev->vm_manager.block_size; 384 if (adev->gmc.translate_further) 385 num_level -= 1; 386 else 387 block_size -= 9; 388 389 inst_mask = adev->aid_mask; 390 for_each_inst(j, inst_mask) { 391 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 392 for (i = 0; i <= 14; i++) { 393 tmp = RREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 394 i * hub->ctx_distance); 395 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 396 ENABLE_CONTEXT, 1); 397 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 398 PAGE_TABLE_DEPTH, num_level); 399 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 400 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 401 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 402 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 403 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 404 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 405 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 406 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 407 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 408 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 409 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 410 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 411 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 412 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 413 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 414 PAGE_TABLE_BLOCK_SIZE, 415 block_size); 416 /* On 9.4.3, XNACK can be enabled in the SQ 417 * per-process. Retry faults need to be enabled for 418 * that to work. 419 */ 420 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 421 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1); 422 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 423 i * hub->ctx_distance, tmp); 424 WREG32_SOC15_OFFSET(MMHUB, j, 425 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 426 i * hub->ctx_addr_distance, 0); 427 WREG32_SOC15_OFFSET(MMHUB, j, 428 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 429 i * hub->ctx_addr_distance, 0); 430 WREG32_SOC15_OFFSET(MMHUB, j, 431 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 432 i * hub->ctx_addr_distance, 433 lower_32_bits(adev->vm_manager.max_pfn - 1)); 434 WREG32_SOC15_OFFSET(MMHUB, j, 435 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 436 i * hub->ctx_addr_distance, 437 upper_32_bits(adev->vm_manager.max_pfn - 1)); 438 } 439 } 440 } 441 442 static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev) 443 { 444 struct amdgpu_vmhub *hub; 445 u32 i, j, inst_mask; 446 447 inst_mask = adev->aid_mask; 448 for_each_inst(j, inst_mask) { 449 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 450 for (i = 0; i < 18; ++i) { 451 WREG32_SOC15_OFFSET(MMHUB, j, 452 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 453 i * hub->eng_addr_distance, 0xffffffff); 454 WREG32_SOC15_OFFSET(MMHUB, j, 455 regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 456 i * hub->eng_addr_distance, 0x1f); 457 } 458 } 459 } 460 461 static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) 462 { 463 /* GART Enable. */ 464 mmhub_v1_8_init_gart_aperture_regs(adev); 465 mmhub_v1_8_init_system_aperture_regs(adev); 466 mmhub_v1_8_init_tlb_regs(adev); 467 mmhub_v1_8_init_cache_regs(adev); 468 mmhub_v1_8_init_snoop_override_regs(adev); 469 470 mmhub_v1_8_enable_system_domain(adev); 471 mmhub_v1_8_disable_identity_aperture(adev); 472 mmhub_v1_8_setup_vmid_config(adev); 473 mmhub_v1_8_program_invalidation(adev); 474 475 return 0; 476 } 477 478 static void mmhub_v1_8_disable_l1_tlb(struct amdgpu_device *adev) 479 { 480 u32 tmp; 481 u32 i, inst_mask; 482 483 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 484 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 485 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 486 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 487 ENABLE_ADVANCED_DRIVER_MODEL, 0); 488 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 489 } else { 490 inst_mask = adev->aid_mask; 491 for_each_inst(i, inst_mask) { 492 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 493 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 494 0); 495 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 496 ENABLE_ADVANCED_DRIVER_MODEL, 0); 497 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 498 } 499 } 500 } 501 502 static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev) 503 { 504 struct amdgpu_vmhub *hub; 505 u32 tmp; 506 u32 i, j, inst_mask; 507 508 /* Disable all tables */ 509 inst_mask = adev->aid_mask; 510 for_each_inst(j, inst_mask) { 511 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 512 for (i = 0; i < 16; i++) 513 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT0_CNTL, 514 i * hub->ctx_distance, 0); 515 if (!amdgpu_sriov_vf(adev)) { 516 /* Setup L2 cache */ 517 tmp = RREG32_SOC15(MMHUB, j, regVM_L2_CNTL); 518 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 519 0); 520 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL, tmp); 521 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL3, 0); 522 } 523 } 524 525 mmhub_v1_8_disable_l1_tlb(adev); 526 } 527 528 /** 529 * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling 530 * 531 * @adev: amdgpu_device pointer 532 * @value: true redirects VM faults to the default page 533 */ 534 static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value) 535 { 536 u32 tmp, inst_mask; 537 int i; 538 539 if (amdgpu_sriov_vf(adev)) 540 return; 541 542 inst_mask = adev->aid_mask; 543 for_each_inst(i, inst_mask) { 544 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL); 545 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 546 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 547 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 548 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 549 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 550 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 551 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 552 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 553 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 554 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 555 value); 556 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 557 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 558 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 559 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 560 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 561 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 562 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 563 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 564 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 565 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 566 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 567 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 568 if (!value) { 569 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 570 CRASH_ON_NO_RETRY_FAULT, 1); 571 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 572 CRASH_ON_RETRY_FAULT, 1); 573 } 574 575 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL, tmp); 576 } 577 } 578 579 static void mmhub_v1_8_init(struct amdgpu_device *adev) 580 { 581 struct amdgpu_vmhub *hub; 582 u32 inst_mask; 583 int i; 584 585 inst_mask = adev->aid_mask; 586 for_each_inst(i, inst_mask) { 587 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 588 589 hub->ctx0_ptb_addr_lo32 = SOC15_REG_OFFSET(MMHUB, i, 590 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 591 hub->ctx0_ptb_addr_hi32 = SOC15_REG_OFFSET(MMHUB, i, 592 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 593 hub->vm_inv_eng0_req = 594 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_REQ); 595 hub->vm_inv_eng0_ack = 596 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_ACK); 597 hub->vm_context0_cntl = 598 SOC15_REG_OFFSET(MMHUB, i, regVM_CONTEXT0_CNTL); 599 hub->vm_l2_pro_fault_status = SOC15_REG_OFFSET(MMHUB, i, 600 regVM_L2_PROTECTION_FAULT_STATUS); 601 hub->vm_l2_pro_fault_cntl = SOC15_REG_OFFSET(MMHUB, i, 602 regVM_L2_PROTECTION_FAULT_CNTL); 603 604 hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL; 605 hub->ctx_addr_distance = 606 regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 607 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 608 hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - 609 regVM_INVALIDATE_ENG0_REQ; 610 hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 611 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 612 } 613 } 614 615 static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev, 616 enum amd_clockgating_state state) 617 { 618 return 0; 619 } 620 621 static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags) 622 { 623 624 } 625 626 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { 627 .get_fb_location = mmhub_v1_8_get_fb_location, 628 .init = mmhub_v1_8_init, 629 .gart_enable = mmhub_v1_8_gart_enable, 630 .set_fault_enable_default = mmhub_v1_8_set_fault_enable_default, 631 .gart_disable = mmhub_v1_8_gart_disable, 632 .setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs, 633 .set_clockgating = mmhub_v1_8_set_clockgating, 634 .get_clockgating = mmhub_v1_8_get_clockgating, 635 }; 636 637 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = { 638 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI), 639 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 640 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI), 641 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 642 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI), 643 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 644 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI), 645 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 646 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI), 647 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 648 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI), 649 1, 0, "MM_CANE"}, 650 }; 651 652 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = { 653 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI), 654 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 655 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI), 656 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 657 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI), 658 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 659 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI), 660 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 661 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI), 662 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 663 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI), 664 1, 0, "MM_CANE"}, 665 }; 666 667 static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = { 668 {AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM"}, 669 {AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM"}, 670 {AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM"}, 671 {AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM"}, 672 {AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM"}, 673 {AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM"}, 674 {AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM"}, 675 {AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM"}, 676 {AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM"}, 677 {AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM"}, 678 {AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0"}, 679 {AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1"}, 680 {AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2"}, 681 {AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3"}, 682 {AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM"}, 683 {AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM"}, 684 {AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM"}, 685 {AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM"}, 686 {AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM"}, 687 }; 688 689 static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev, 690 uint32_t mmhub_inst, 691 void *ras_err_status) 692 { 693 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 694 unsigned long ue_count = 0, ce_count = 0; 695 696 /* NOTE: mmhub is converted by aid_mask and the range is 0-3, 697 * which can be used as die ID directly */ 698 struct amdgpu_smuio_mcm_config_info mcm_info = { 699 .socket_id = adev->smuio.funcs->get_socket_id(adev), 700 .die_id = mmhub_inst, 701 }; 702 703 amdgpu_ras_inst_query_ras_error_count(adev, 704 mmhub_v1_8_ce_reg_list, 705 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 706 mmhub_v1_8_ras_memory_list, 707 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 708 mmhub_inst, 709 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, 710 &ce_count); 711 amdgpu_ras_inst_query_ras_error_count(adev, 712 mmhub_v1_8_ue_reg_list, 713 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 714 mmhub_v1_8_ras_memory_list, 715 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 716 mmhub_inst, 717 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 718 &ue_count); 719 720 amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count); 721 amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); 722 } 723 724 static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev, 725 void *ras_err_status) 726 { 727 uint32_t inst_mask; 728 uint32_t i; 729 730 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 731 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 732 return; 733 } 734 735 inst_mask = adev->aid_mask; 736 for_each_inst(i, inst_mask) 737 mmhub_v1_8_inst_query_ras_error_count(adev, i, ras_err_status); 738 } 739 740 static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev, 741 uint32_t mmhub_inst) 742 { 743 amdgpu_ras_inst_reset_ras_error_count(adev, 744 mmhub_v1_8_ce_reg_list, 745 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 746 mmhub_inst); 747 amdgpu_ras_inst_reset_ras_error_count(adev, 748 mmhub_v1_8_ue_reg_list, 749 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 750 mmhub_inst); 751 } 752 753 static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev) 754 { 755 uint32_t inst_mask; 756 uint32_t i; 757 758 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 759 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 760 return; 761 } 762 763 inst_mask = adev->aid_mask; 764 for_each_inst(i, inst_mask) 765 mmhub_v1_8_inst_reset_ras_error_count(adev, i); 766 } 767 768 static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = { 769 .query_ras_error_count = mmhub_v1_8_query_ras_error_count, 770 .reset_ras_error_count = mmhub_v1_8_reset_ras_error_count, 771 }; 772 773 static int mmhub_v1_8_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 774 enum aca_smu_type type, void *data) 775 { 776 struct aca_bank_info info; 777 u64 misc0; 778 int ret; 779 780 ret = aca_bank_info_decode(bank, &info); 781 if (ret) 782 return ret; 783 784 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 785 switch (type) { 786 case ACA_SMU_TYPE_UE: 787 bank->aca_err_type = ACA_ERROR_TYPE_UE; 788 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 789 1ULL); 790 break; 791 case ACA_SMU_TYPE_CE: 792 bank->aca_err_type = ACA_ERROR_TYPE_CE; 793 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 794 ACA_REG__MISC0__ERRCNT(misc0)); 795 break; 796 default: 797 return -EINVAL; 798 } 799 800 return ret; 801 } 802 803 /* reference to smu driver if header file */ 804 static int mmhub_v1_8_err_codes[] = { 805 0, 1, 2, 3, 4, /* CODE_DAGB0 - 4 */ 806 5, 6, 7, 8, 9, /* CODE_EA0 - 4 */ 807 10, /* CODE_UTCL2_ROUTER */ 808 11, /* CODE_VML2 */ 809 12, /* CODE_VML2_WALKER */ 810 13, /* CODE_MMCANE */ 811 }; 812 813 static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 814 enum aca_smu_type type, void *data) 815 { 816 u32 instlo; 817 818 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 819 instlo &= GENMASK(31, 1); 820 821 if (instlo != mmSMNAID_AID0_MCA_SMU) 822 return false; 823 824 if (aca_bank_check_error_codes(handle->adev, bank, 825 mmhub_v1_8_err_codes, 826 ARRAY_SIZE(mmhub_v1_8_err_codes))) 827 return false; 828 829 return true; 830 } 831 832 static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = { 833 .aca_bank_parser = mmhub_v1_8_aca_bank_parser, 834 .aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid, 835 }; 836 837 static const struct aca_info mmhub_v1_8_aca_info = { 838 .hwip = ACA_HWIP_TYPE_SMU, 839 .mask = ACA_ERROR_UE_MASK, 840 .bank_ops = &mmhub_v1_8_aca_bank_ops, 841 }; 842 843 static int mmhub_v1_8_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 844 { 845 int r; 846 847 r = amdgpu_ras_block_late_init(adev, ras_block); 848 if (r) 849 return r; 850 851 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__MMHUB, 852 &mmhub_v1_8_aca_info, NULL); 853 if (r) 854 goto late_fini; 855 856 return 0; 857 858 late_fini: 859 amdgpu_ras_block_late_fini(adev, ras_block); 860 861 return r; 862 } 863 864 struct amdgpu_mmhub_ras mmhub_v1_8_ras = { 865 .ras_block = { 866 .hw_ops = &mmhub_v1_8_ras_hw_ops, 867 .ras_late_init = mmhub_v1_8_ras_late_init, 868 }, 869 }; 870