Lines Matching +full:gfx +full:- +full:mem
35 #include <linux/pci-p2pdma.h>
36 #include <linux/apple-gmux.h>
85 #include <asm/intel-family.h>
98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
196 return -EINVAL; in amdgpu_sysfs_reg_state_get()
212 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_init()
221 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_fini()
234 * - "cem" - PCIE CEM card
235 * - "oam" - Open Compute Accelerator Module
236 * - "unknown" - Not known
249 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) in amdgpu_device_get_board_info()
250 pkg_type = adev->smuio.funcs->get_pkg_type(adev); in amdgpu_device_get_board_info()
281 if (adev->flags & AMD_IS_APU) in amdgpu_board_attrs_is_visible()
284 return attr->mode; in amdgpu_board_attrs_is_visible()
296 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
307 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) in amdgpu_device_supports_px()
313 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
324 if (adev->has_pr3 || in amdgpu_device_supports_boco()
325 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) in amdgpu_device_supports_boco()
331 * amdgpu_device_supports_baco - Does the device support BACO
346 * amdgpu_device_supports_smart_shift - Is the device dGPU with
365 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
371 * @write: true - write to vram, otherwise - read from vram
387 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
402 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
407 * amdgpu_device_aper_access - access vram by vram aperature
413 * @write: true - write to vram, otherwise - read from vram
425 if (!adev->mman.aper_base_kaddr) in amdgpu_device_aper_access()
428 last = min(pos + size, adev->gmc.visible_vram_size); in amdgpu_device_aper_access()
430 addr = adev->mman.aper_base_kaddr + pos; in amdgpu_device_aper_access()
431 count = last - pos; in amdgpu_device_aper_access()
458 * amdgpu_device_vram_access - read/write a buffer in vram
464 * @write: true - write to vram, otherwise - read from vram
473 size -= count; in amdgpu_device_vram_access()
489 if (adev->no_hw_access) in amdgpu_device_skip_hw_access()
505 if (down_read_trylock(&adev->reset_domain->sem)) in amdgpu_device_skip_hw_access()
506 up_read(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
508 lockdep_assert_held(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
515 * amdgpu_device_rreg - read a memory mapped IO or indirect register
531 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_rreg()
534 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_rreg()
536 up_read(&adev->reset_domain->sem); in amdgpu_device_rreg()
538 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_rreg()
541 ret = adev->pcie_rreg(adev, reg * 4); in amdgpu_device_rreg()
544 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); in amdgpu_device_rreg()
555 * amdgpu_mm_rreg8 - read a memory mapped IO register
567 if (offset < adev->rmmio_size) in amdgpu_mm_rreg8()
568 return (readb(adev->rmmio + offset)); in amdgpu_mm_rreg8()
574 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
592 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_xcc_rreg()
595 adev->gfx.rlc.rlcg_reg_access_supported && in amdgpu_device_xcc_rreg()
602 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_xcc_rreg()
604 up_read(&adev->reset_domain->sem); in amdgpu_device_xcc_rreg()
606 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_xcc_rreg()
609 ret = adev->pcie_rreg(adev, reg * 4); in amdgpu_device_xcc_rreg()
622 * amdgpu_mm_wreg8 - read a memory mapped IO register
635 if (offset < adev->rmmio_size) in amdgpu_mm_wreg8()
636 writeb(value, adev->rmmio + offset); in amdgpu_mm_wreg8()
642 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
658 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_wreg()
661 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_wreg()
663 up_read(&adev->reset_domain->sem); in amdgpu_device_wreg()
665 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_wreg()
668 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_device_wreg()
671 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); in amdgpu_device_wreg()
675 …* amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if i…
692 adev->gfx.rlc.funcs && in amdgpu_mm_wreg_mmio_rlc()
693 adev->gfx.rlc.funcs->is_rlcg_access_range) { in amdgpu_mm_wreg_mmio_rlc()
694 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) in amdgpu_mm_wreg_mmio_rlc()
696 } else if ((reg * 4) >= adev->rmmio_size) { in amdgpu_mm_wreg_mmio_rlc()
697 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_mm_wreg_mmio_rlc()
699 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_wreg_mmio_rlc()
704 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
723 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_xcc_wreg()
726 adev->gfx.rlc.rlcg_reg_access_supported && in amdgpu_device_xcc_wreg()
733 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_xcc_wreg()
735 up_read(&adev->reset_domain->sem); in amdgpu_device_xcc_wreg()
737 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_xcc_wreg()
740 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_device_xcc_wreg()
745 * amdgpu_device_indirect_rreg - read an indirect register
760 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg()
761 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg()
763 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg()
764 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg()
765 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg()
770 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg()
784 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg_ext()
785 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg_ext()
786 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_rreg_ext()
787 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_rreg_ext()
791 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg_ext()
792 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg_ext()
793 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg_ext()
795 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_rreg_ext()
812 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg_ext()
818 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
833 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg64()
834 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg64()
836 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64()
837 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg64()
838 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg64()
848 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64()
863 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg64_ext()
864 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg64_ext()
865 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_rreg64_ext()
866 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_rreg64_ext()
868 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64_ext()
869 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg64_ext()
870 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg64_ext()
872 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_rreg64_ext()
898 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64_ext()
904 * amdgpu_device_indirect_wreg - write an indirect register address
918 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg()
919 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg()
921 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg()
922 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg()
923 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg()
929 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg()
940 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg_ext()
941 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg_ext()
942 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_wreg_ext()
943 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_wreg_ext()
947 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg_ext()
948 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg_ext()
949 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg_ext()
951 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_wreg_ext()
969 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg_ext()
973 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
987 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg64()
988 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg64()
990 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64()
991 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg64()
992 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg64()
1004 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64()
1016 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1017 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1018 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_wreg64_ext()
1019 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1021 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64_ext()
1022 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg64_ext()
1023 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg64_ext()
1025 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_wreg64_ext()
1053 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64_ext()
1057 * amdgpu_device_get_rev_id - query device rev_id
1065 return adev->nbio.funcs->get_rev_id(adev); in amdgpu_device_get_rev_id()
1069 * amdgpu_invalid_rreg - dummy reg read function
1093 * amdgpu_invalid_wreg - dummy reg write function
1117 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1141 * amdgpu_invalid_wreg64 - dummy reg write function
1165 * amdgpu_block_invalid_rreg - dummy reg read function
1185 * amdgpu_block_invalid_wreg - dummy reg write function
1205 * amdgpu_device_asic_init - Wrapper for atom asic_init
1225 return amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_asic_init()
1232 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1244 &adev->mem_scratch.robj, in amdgpu_device_mem_scratch_init()
1245 &adev->mem_scratch.gpu_addr, in amdgpu_device_mem_scratch_init()
1246 (void **)&adev->mem_scratch.ptr); in amdgpu_device_mem_scratch_init()
1250 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1258 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); in amdgpu_device_mem_scratch_fini()
1262 * amdgpu_device_program_register_sequence - program an array of registers.
1291 if (adev->family >= AMDGPU_FAMILY_AI) in amdgpu_device_program_register_sequence()
1301 * amdgpu_device_pci_config_reset - reset the GPU
1310 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); in amdgpu_device_pci_config_reset()
1314 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1322 return pci_reset_function(adev->pdev); in amdgpu_device_pci_reset()
1332 * amdgpu_device_wb_fini - Disable Writeback and free memory
1341 if (adev->wb.wb_obj) { in amdgpu_device_wb_fini()
1342 amdgpu_bo_free_kernel(&adev->wb.wb_obj, in amdgpu_device_wb_fini()
1343 &adev->wb.gpu_addr, in amdgpu_device_wb_fini()
1344 (void **)&adev->wb.wb); in amdgpu_device_wb_fini()
1345 adev->wb.wb_obj = NULL; in amdgpu_device_wb_fini()
1350 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1356 * Returns 0 on success or an -error on failure.
1362 if (adev->wb.wb_obj == NULL) { in amdgpu_device_wb_init()
1366 &adev->wb.wb_obj, &adev->wb.gpu_addr, in amdgpu_device_wb_init()
1367 (void **)&adev->wb.wb); in amdgpu_device_wb_init()
1369 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); in amdgpu_device_wb_init()
1373 adev->wb.num_wb = AMDGPU_MAX_WB; in amdgpu_device_wb_init()
1374 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); in amdgpu_device_wb_init()
1377 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); in amdgpu_device_wb_init()
1384 * amdgpu_device_wb_get - Allocate a wb entry
1390 * Returns 0 on success or -EINVAL on failure.
1394 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); in amdgpu_device_wb_get()
1396 if (offset < adev->wb.num_wb) { in amdgpu_device_wb_get()
1397 __set_bit(offset, adev->wb.used); in amdgpu_device_wb_get()
1401 return -EINVAL; in amdgpu_device_wb_get()
1406 * amdgpu_device_wb_free - Free a wb entry
1416 if (wb < adev->wb.num_wb) in amdgpu_device_wb_free()
1417 __clear_bit(wb, adev->wb.used); in amdgpu_device_wb_free()
1421 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1427 * driver loading by returning -ENODEV.
1431 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); in amdgpu_device_resize_fb_bar()
1446 if (adev->gmc.real_vram_size && in amdgpu_device_resize_fb_bar()
1447 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) in amdgpu_device_resize_fb_bar()
1451 root = adev->pdev->bus; in amdgpu_device_resize_fb_bar()
1452 while (root->parent) in amdgpu_device_resize_fb_bar()
1453 root = root->parent; in amdgpu_device_resize_fb_bar()
1456 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && in amdgpu_device_resize_fb_bar()
1457 res->start > 0x100000000ull) in amdgpu_device_resize_fb_bar()
1466 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, in amdgpu_device_resize_fb_bar()
1470 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); in amdgpu_device_resize_fb_bar()
1471 pci_write_config_word(adev->pdev, PCI_COMMAND, in amdgpu_device_resize_fb_bar()
1476 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_resize_fb_bar()
1477 pci_release_resource(adev->pdev, 2); in amdgpu_device_resize_fb_bar()
1479 pci_release_resource(adev->pdev, 0); in amdgpu_device_resize_fb_bar()
1481 r = pci_resize_resource(adev->pdev, 0, rbar_size); in amdgpu_device_resize_fb_bar()
1482 if (r == -ENOSPC) in amdgpu_device_resize_fb_bar()
1484 else if (r && r != -ENOTSUPP) in amdgpu_device_resize_fb_bar()
1487 pci_assign_unassigned_bus_resources(adev->pdev->bus); in amdgpu_device_resize_fb_bar()
1493 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) in amdgpu_device_resize_fb_bar()
1494 return -ENODEV; in amdgpu_device_resize_fb_bar()
1496 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); in amdgpu_device_resize_fb_bar()
1503 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) in amdgpu_device_read_bios()
1513 * amdgpu_device_need_post - check if the hw need post or not
1532 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot in amdgpu_device_need_post()
1537 if (adev->asic_type == CHIP_FIJI) { in amdgpu_device_need_post()
1541 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); in amdgpu_device_need_post()
1546 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); in amdgpu_device_need_post()
1547 release_firmware(adev->pm.fw); in amdgpu_device_need_post()
1554 if (adev->gmc.xgmi.pending_reset) in amdgpu_device_need_post()
1557 if (adev->has_hw_reset) { in amdgpu_device_need_post()
1558 adev->has_hw_reset = false; in amdgpu_device_need_post()
1563 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_need_post()
1585 case -1: in amdgpu_device_seamless_boot_supported()
1597 if (!(adev->flags & AMD_IS_APU)) in amdgpu_device_seamless_boot_supported()
1600 if (adev->mman.keep_stolen_vga_memory) in amdgpu_device_seamless_boot_supported()
1611 …gn/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-…
1612 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1620 if (dev_is_removable(adev->dev)) in amdgpu_device_pcie_dynamic_switching_supported()
1623 if (c->x86_vendor == X86_VENDOR_INTEL) in amdgpu_device_pcie_dynamic_switching_supported()
1630 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1642 case -1: in amdgpu_device_should_use_aspm()
1651 if (adev->flags & AMD_IS_APU) in amdgpu_device_should_use_aspm()
1653 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) in amdgpu_device_should_use_aspm()
1655 return pcie_aspm_enabled(adev->pdev); in amdgpu_device_should_use_aspm()
1660 * amdgpu_device_vga_set_decode - enable/disable vga decode
1682 * amdgpu_device_check_block_size - validate the vm block size
1697 if (amdgpu_vm_block_size == -1) in amdgpu_device_check_block_size()
1701 dev_warn(adev->dev, "VM page table size (%d) too small\n", in amdgpu_device_check_block_size()
1703 amdgpu_vm_block_size = -1; in amdgpu_device_check_block_size()
1708 * amdgpu_device_check_vm_size - validate the vm size
1718 if (amdgpu_vm_size == -1) in amdgpu_device_check_vm_size()
1722 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", in amdgpu_device_check_vm_size()
1724 amdgpu_vm_size = -1; in amdgpu_device_check_vm_size()
1740 DRM_WARN("Not 64-bit OS, feature not supported\n"); in amdgpu_device_check_smu_prv_buffer_size()
1758 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; in amdgpu_device_check_smu_prv_buffer_size()
1765 adev->pm.smu_prv_buffer_size = 0; in amdgpu_device_check_smu_prv_buffer_size()
1770 if (!(adev->flags & AMD_IS_APU) || in amdgpu_device_init_apu_flags()
1771 adev->asic_type < CHIP_RAVEN) in amdgpu_device_init_apu_flags()
1774 switch (adev->asic_type) { in amdgpu_device_init_apu_flags()
1776 if (adev->pdev->device == 0x15dd) in amdgpu_device_init_apu_flags()
1777 adev->apu_flags |= AMD_APU_IS_RAVEN; in amdgpu_device_init_apu_flags()
1778 if (adev->pdev->device == 0x15d8) in amdgpu_device_init_apu_flags()
1779 adev->apu_flags |= AMD_APU_IS_PICASSO; in amdgpu_device_init_apu_flags()
1782 if ((adev->pdev->device == 0x1636) || in amdgpu_device_init_apu_flags()
1783 (adev->pdev->device == 0x164c)) in amdgpu_device_init_apu_flags()
1784 adev->apu_flags |= AMD_APU_IS_RENOIR; in amdgpu_device_init_apu_flags()
1786 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; in amdgpu_device_init_apu_flags()
1789 adev->apu_flags |= AMD_APU_IS_VANGOGH; in amdgpu_device_init_apu_flags()
1794 if ((adev->pdev->device == 0x13FE) || in amdgpu_device_init_apu_flags()
1795 (adev->pdev->device == 0x143F)) in amdgpu_device_init_apu_flags()
1796 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; in amdgpu_device_init_apu_flags()
1806 * amdgpu_device_check_arguments - validate module params
1816 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", in amdgpu_device_check_arguments()
1820 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1825 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { in amdgpu_device_check_arguments()
1827 dev_warn(adev->dev, "gart size (%d) too small\n", in amdgpu_device_check_arguments()
1829 amdgpu_gart_size = -1; in amdgpu_device_check_arguments()
1832 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { in amdgpu_device_check_arguments()
1834 dev_warn(adev->dev, "gtt size (%d) too small\n", in amdgpu_device_check_arguments()
1836 amdgpu_gtt_size = -1; in amdgpu_device_check_arguments()
1840 if (amdgpu_vm_fragment_size != -1 && in amdgpu_device_check_arguments()
1842 dev_warn(adev->dev, "valid range is between 4 and 9\n"); in amdgpu_device_check_arguments()
1843 amdgpu_vm_fragment_size = -1; in amdgpu_device_check_arguments()
1847 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", in amdgpu_device_check_arguments()
1851 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1856 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { in amdgpu_device_check_arguments()
1857 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); in amdgpu_device_check_arguments()
1858 amdgpu_reset_method = -1; in amdgpu_device_check_arguments()
1867 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); in amdgpu_device_check_arguments()
1873 * amdgpu_switcheroo_set_state - set switcheroo state
1893 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
1902 dev->switch_power_state = DRM_SWITCH_POWER_ON; in amdgpu_switcheroo_set_state()
1905 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
1912 dev->switch_power_state = DRM_SWITCH_POWER_OFF; in amdgpu_switcheroo_set_state()
1917 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1934 return atomic_read(&dev->open_count) == 0; in amdgpu_switcheroo_can_switch()
1944 * amdgpu_device_ip_set_clockgating_state - set the CG state
1947 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1961 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_clockgating_state()
1962 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_clockgating_state()
1964 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_clockgating_state()
1966 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) in amdgpu_device_ip_set_clockgating_state()
1968 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( in amdgpu_device_ip_set_clockgating_state()
1972 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_clockgating_state()
1978 * amdgpu_device_ip_set_powergating_state - set the PG state
1981 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1995 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_powergating_state()
1996 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_powergating_state()
1998 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_powergating_state()
2000 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) in amdgpu_device_ip_set_powergating_state()
2002 r = adev->ip_blocks[i].version->funcs->set_powergating_state( in amdgpu_device_ip_set_powergating_state()
2006 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_powergating_state()
2012 * amdgpu_device_ip_get_clockgating_state - get the CG state
2027 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_get_clockgating_state()
2028 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_get_clockgating_state()
2030 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) in amdgpu_device_ip_get_clockgating_state()
2031 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); in amdgpu_device_ip_get_clockgating_state()
2036 * amdgpu_device_ip_wait_for_idle - wait for idle
2039 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2049 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_wait_for_idle()
2050 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_wait_for_idle()
2052 if (adev->ip_blocks[i].version->type == block_type) { in amdgpu_device_ip_wait_for_idle()
2053 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); in amdgpu_device_ip_wait_for_idle()
2064 * amdgpu_device_ip_is_idle - is the hardware IP idle
2067 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2077 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_is_idle()
2078 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_is_idle()
2080 if (adev->ip_blocks[i].version->type == block_type) in amdgpu_device_ip_is_idle()
2081 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); in amdgpu_device_ip_is_idle()
2088 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2091 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2102 for (i = 0; i < adev->num_ip_blocks; i++) in amdgpu_device_ip_get_ip_block()
2103 if (adev->ip_blocks[i].version->type == type) in amdgpu_device_ip_get_ip_block()
2104 return &adev->ip_blocks[i]; in amdgpu_device_ip_get_ip_block()
2126 if (ip_block && ((ip_block->version->major > major) || in amdgpu_device_ip_block_version_cmp()
2127 ((ip_block->version->major == major) && in amdgpu_device_ip_block_version_cmp()
2128 (ip_block->version->minor >= minor)))) in amdgpu_device_ip_block_version_cmp()
2147 return -EINVAL; in amdgpu_device_ip_block_add()
2149 switch (ip_block_version->type) { in amdgpu_device_ip_block_add()
2151 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) in amdgpu_device_ip_block_add()
2155 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) in amdgpu_device_ip_block_add()
2162 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, in amdgpu_device_ip_block_add()
2163 ip_block_version->funcs->name); in amdgpu_device_ip_block_add()
2165 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; in amdgpu_device_ip_block_add()
2171 * amdgpu_device_enable_virtual_display - enable virtual display feature
2184 adev->enable_virtual_display = false; in amdgpu_device_enable_virtual_display()
2187 const char *pci_address_name = pci_name(adev->pdev); in amdgpu_device_enable_virtual_display()
2197 int res = -1; in amdgpu_device_enable_virtual_display()
2199 adev->enable_virtual_display = true; in amdgpu_device_enable_virtual_display()
2210 adev->mode_info.num_crtc = num_crtc; in amdgpu_device_enable_virtual_display()
2212 adev->mode_info.num_crtc = 1; in amdgpu_device_enable_virtual_display()
2220 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_enable_virtual_display()
2228 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { in amdgpu_device_set_sriov_virtual_display()
2229 adev->mode_info.num_crtc = 1; in amdgpu_device_set_sriov_virtual_display()
2230 adev->enable_virtual_display = true; in amdgpu_device_set_sriov_virtual_display()
2232 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_set_sriov_virtual_display()
2237 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2244 * Returns 0 on success, -EINVAL on failure.
2253 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_parse_gpu_info_fw()
2255 if (adev->mman.discovery_bin) in amdgpu_device_parse_gpu_info_fw()
2258 switch (adev->asic_type) { in amdgpu_device_parse_gpu_info_fw()
2268 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in amdgpu_device_parse_gpu_info_fw()
2270 else if (adev->apu_flags & AMD_APU_IS_PICASSO) in amdgpu_device_parse_gpu_info_fw()
2284 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name); in amdgpu_device_parse_gpu_info_fw()
2286 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
2292 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; in amdgpu_device_parse_gpu_info_fw()
2293 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); in amdgpu_device_parse_gpu_info_fw()
2295 switch (hdr->version_major) { in amdgpu_device_parse_gpu_info_fw()
2299 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2300 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2305 if (adev->asic_type == CHIP_NAVI12) in amdgpu_device_parse_gpu_info_fw()
2308 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); in amdgpu_device_parse_gpu_info_fw()
2309 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); in amdgpu_device_parse_gpu_info_fw()
2310 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); in amdgpu_device_parse_gpu_info_fw()
2311 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); in amdgpu_device_parse_gpu_info_fw()
2312 adev->gfx.config.max_texture_channel_caches = in amdgpu_device_parse_gpu_info_fw()
2313 le32_to_cpu(gpu_info_fw->gc_num_tccs); in amdgpu_device_parse_gpu_info_fw()
2314 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); in amdgpu_device_parse_gpu_info_fw()
2315 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); in amdgpu_device_parse_gpu_info_fw()
2316 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); in amdgpu_device_parse_gpu_info_fw()
2317 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); in amdgpu_device_parse_gpu_info_fw()
2318 adev->gfx.config.double_offchip_lds_buf = in amdgpu_device_parse_gpu_info_fw()
2319 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); in amdgpu_device_parse_gpu_info_fw()
2320 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); in amdgpu_device_parse_gpu_info_fw()
2321 adev->gfx.cu_info.max_waves_per_simd = in amdgpu_device_parse_gpu_info_fw()
2322 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); in amdgpu_device_parse_gpu_info_fw()
2323 adev->gfx.cu_info.max_scratch_slots_per_cu = in amdgpu_device_parse_gpu_info_fw()
2324 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); in amdgpu_device_parse_gpu_info_fw()
2325 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); in amdgpu_device_parse_gpu_info_fw()
2326 if (hdr->version_minor >= 1) { in amdgpu_device_parse_gpu_info_fw()
2328 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2329 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2330 adev->gfx.config.num_sc_per_sh = in amdgpu_device_parse_gpu_info_fw()
2331 le32_to_cpu(gpu_info_fw->num_sc_per_sh); in amdgpu_device_parse_gpu_info_fw()
2332 adev->gfx.config.num_packer_per_sc = in amdgpu_device_parse_gpu_info_fw()
2333 le32_to_cpu(gpu_info_fw->num_packer_per_sc); in amdgpu_device_parse_gpu_info_fw()
2341 if (hdr->version_minor == 2) { in amdgpu_device_parse_gpu_info_fw()
2343 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2344 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2345 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; in amdgpu_device_parse_gpu_info_fw()
2350 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
2351 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); in amdgpu_device_parse_gpu_info_fw()
2352 err = -EINVAL; in amdgpu_device_parse_gpu_info_fw()
2360 * amdgpu_device_ip_early_init - run early init for hardware IPs
2383 switch (adev->asic_type) { in amdgpu_device_ip_early_init()
2390 adev->family = AMDGPU_FAMILY_SI; in amdgpu_device_ip_early_init()
2402 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
2403 adev->family = AMDGPU_FAMILY_KV; in amdgpu_device_ip_early_init()
2405 adev->family = AMDGPU_FAMILY_CI; in amdgpu_device_ip_early_init()
2421 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
2422 adev->family = AMDGPU_FAMILY_CZ; in amdgpu_device_ip_early_init()
2424 adev->family = AMDGPU_FAMILY_VI; in amdgpu_device_ip_early_init()
2440 ((adev->flags & AMD_IS_APU) == 0) && in amdgpu_device_ip_early_init()
2441 !dev_is_removable(&adev->pdev->dev)) in amdgpu_device_ip_early_init()
2442 adev->flags |= AMD_IS_PX; in amdgpu_device_ip_early_init()
2444 if (!(adev->flags & AMD_IS_APU)) { in amdgpu_device_ip_early_init()
2445 parent = pcie_find_root_port(adev->pdev); in amdgpu_device_ip_early_init()
2446 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; in amdgpu_device_ip_early_init()
2450 adev->pm.pp_feature = amdgpu_pp_feature_mask; in amdgpu_device_ip_early_init()
2452 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in amdgpu_device_ip_early_init()
2453 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) in amdgpu_device_ip_early_init()
2454 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; in amdgpu_device_ip_early_init()
2456 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; in amdgpu_device_ip_early_init()
2459 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_early_init()
2462 i, adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_early_init()
2463 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2465 if (adev->ip_blocks[i].version->funcs->early_init) { in amdgpu_device_ip_early_init()
2466 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); in amdgpu_device_ip_early_init()
2467 if (r == -ENOENT) { in amdgpu_device_ip_early_init()
2468 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2471 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_early_init()
2474 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2477 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2481 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_early_init()
2489 return -EINVAL; in amdgpu_device_ip_early_init()
2493 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); in amdgpu_device_ip_early_init()
2506 return -ENODEV; in amdgpu_device_ip_early_init()
2509 adev->cg_flags &= amdgpu_cg_mask; in amdgpu_device_ip_early_init()
2510 adev->pg_flags &= amdgpu_pg_mask; in amdgpu_device_ip_early_init()
2519 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase1()
2520 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase1()
2522 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase1()
2524 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_hw_init_phase1()
2525 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || in amdgpu_device_ip_hw_init_phase1()
2526 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_hw_init_phase1()
2527 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase1()
2530 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase1()
2533 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase1()
2544 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase2()
2545 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase2()
2547 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase2()
2549 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase2()
2552 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase2()
2555 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase2()
2567 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_device_fw_loading()
2568 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_fw_loading()
2569 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_fw_loading()
2572 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_fw_loading()
2576 if (adev->ip_blocks[i].status.hw == true) in amdgpu_device_fw_loading()
2579 if (amdgpu_in_reset(adev) || adev->in_suspend) { in amdgpu_device_fw_loading()
2580 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_fw_loading()
2583 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
2587 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_fw_loading()
2590 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
2595 adev->ip_blocks[i].status.hw = true; in amdgpu_device_fw_loading()
2600 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) in amdgpu_device_fw_loading()
2612 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_init_schedulers()
2615 if (!ring || ring->no_scheduler) in amdgpu_device_init_schedulers()
2618 switch (ring->funcs->type) { in amdgpu_device_init_schedulers()
2620 timeout = adev->gfx_timeout; in amdgpu_device_init_schedulers()
2623 timeout = adev->compute_timeout; in amdgpu_device_init_schedulers()
2626 timeout = adev->sdma_timeout; in amdgpu_device_init_schedulers()
2629 timeout = adev->video_timeout; in amdgpu_device_init_schedulers()
2633 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL, in amdgpu_device_init_schedulers()
2635 ring->num_hw_submission, 0, in amdgpu_device_init_schedulers()
2636 timeout, adev->reset_domain->wq, in amdgpu_device_init_schedulers()
2637 ring->sched_score, ring->name, in amdgpu_device_init_schedulers()
2638 adev->dev); in amdgpu_device_init_schedulers()
2641 ring->name); in amdgpu_device_init_schedulers()
2647 ring->name); in amdgpu_device_init_schedulers()
2653 ring->name); in amdgpu_device_init_schedulers()
2665 * amdgpu_device_ip_init - run init for hardware IPs
2683 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
2684 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_init()
2686 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); in amdgpu_device_ip_init()
2689 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_init()
2692 adev->ip_blocks[i].status.sw = true; in amdgpu_device_ip_init()
2694 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_init()
2696 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
2701 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2702 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_init()
2703 /* need to do gmc hw init early so we can allocate gpu mem */ in amdgpu_device_ip_init()
2713 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
2723 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2726 if (adev->gfx.mcbp) { in amdgpu_device_ip_init()
2727 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, in amdgpu_device_ip_init()
2750 dev_err(adev->dev, "IB initialization failed (%d).\n", r); in amdgpu_device_ip_init()
2793 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_init()
2799 r = -ENOENT; in amdgpu_device_ip_init()
2803 if (!hive->reset_domain || in amdgpu_device_ip_init()
2804 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { in amdgpu_device_ip_init()
2805 r = -ENOENT; in amdgpu_device_ip_init()
2811 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_ip_init()
2812 adev->reset_domain = hive->reset_domain; in amdgpu_device_ip_init()
2822 if (adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_device_ip_init()
2826 if (!adev->gmc.xgmi.pending_reset) { in amdgpu_device_ip_init()
2839 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2849 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); in amdgpu_device_fill_reset_magic()
2853 * amdgpu_device_check_vram_lost - check if vram is valid
2864 if (memcmp(adev->gart.ptr, adev->reset_magic, in amdgpu_device_check_vram_lost()
2885 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2905 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_cg_state()
2906 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_cg_state()
2907 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_cg_state()
2909 /* skip CG for GFX, SDMA on S0ix */ in amdgpu_device_set_cg_state()
2910 if (adev->in_s0ix && in amdgpu_device_set_cg_state()
2911 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_cg_state()
2912 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_cg_state()
2915 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_cg_state()
2916 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_cg_state()
2917 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_cg_state()
2918 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_cg_state()
2919 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_set_cg_state()
2921 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_set_cg_state()
2925 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_cg_state()
2942 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_pg_state()
2943 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_pg_state()
2944 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_pg_state()
2946 /* skip PG for GFX, SDMA on S0ix */ in amdgpu_device_set_pg_state()
2947 if (adev->in_s0ix && in amdgpu_device_set_pg_state()
2948 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_pg_state()
2949 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_pg_state()
2952 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_pg_state()
2953 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_pg_state()
2954 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_pg_state()
2955 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_pg_state()
2956 adev->ip_blocks[i].version->funcs->set_powergating_state) { in amdgpu_device_set_pg_state()
2958 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, in amdgpu_device_set_pg_state()
2962 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_pg_state()
2988 adev = gpu_ins->adev; in amdgpu_device_enable_mgpu_fan_boost()
2989 if (!(adev->flags & AMD_IS_APU) && in amdgpu_device_enable_mgpu_fan_boost()
2990 !gpu_ins->mgpu_fan_enabled) { in amdgpu_device_enable_mgpu_fan_boost()
2995 gpu_ins->mgpu_fan_enabled = 1; in amdgpu_device_enable_mgpu_fan_boost()
3006 * amdgpu_device_ip_late_init - run late init for hardware IPs
3022 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_init()
3023 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_late_init()
3025 if (adev->ip_blocks[i].version->funcs->late_init) { in amdgpu_device_ip_late_init()
3026 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); in amdgpu_device_ip_late_init()
3029 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_init()
3033 adev->ip_blocks[i].status.late_initialized = true; in amdgpu_device_ip_late_init()
3055 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || in amdgpu_device_ip_late_init()
3056 adev->asic_type == CHIP_ALDEBARAN)) in amdgpu_device_ip_late_init()
3059 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_late_init()
3063 * Reset device p-state to low as this was booted with high. in amdgpu_device_ip_late_init()
3075 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_ip_late_init()
3078 if (gpu_instance->adev->flags & AMD_IS_APU) in amdgpu_device_ip_late_init()
3081 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, in amdgpu_device_ip_late_init()
3097 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3110 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_smu_fini_early()
3111 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_smu_fini_early()
3113 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_smu_fini_early()
3114 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_smu_fini_early()
3118 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_smu_fini_early()
3120 adev->ip_blocks[i].status.hw = false; in amdgpu_device_smu_fini_early()
3130 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_fini_early()
3131 if (!adev->ip_blocks[i].version->funcs->early_fini) in amdgpu_device_ip_fini_early()
3134 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev); in amdgpu_device_ip_fini_early()
3137 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini_early()
3149 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini_early()
3150 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini_early()
3153 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini_early()
3157 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini_early()
3160 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini_early()
3172 * amdgpu_device_ip_fini - run fini for hardware IPs
3186 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) in amdgpu_device_ip_fini()
3189 if (adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_ip_fini()
3194 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
3195 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_fini()
3198 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_fini()
3200 amdgpu_free_static_csa(&adev->virt.csa_obj); in amdgpu_device_ip_fini()
3207 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); in amdgpu_device_ip_fini()
3211 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
3213 adev->ip_blocks[i].status.sw = false; in amdgpu_device_ip_fini()
3214 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_fini()
3217 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
3218 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_ip_fini()
3220 if (adev->ip_blocks[i].version->funcs->late_fini) in amdgpu_device_ip_fini()
3221 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); in amdgpu_device_ip_fini()
3222 adev->ip_blocks[i].status.late_initialized = false; in amdgpu_device_ip_fini()
3231 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3249 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); in amdgpu_device_delay_enable_gfx_off()
3251 WARN_ON_ONCE(adev->gfx.gfx_off_state); in amdgpu_device_delay_enable_gfx_off()
3252 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); in amdgpu_device_delay_enable_gfx_off()
3255 adev->gfx.gfx_off_state = true; in amdgpu_device_delay_enable_gfx_off()
3259 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3282 dev_warn(adev->dev, "Failed to disallow df cstate"); in amdgpu_device_ip_suspend_phase1()
3284 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase1()
3285 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase1()
3289 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase1()
3293 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase1()
3297 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase1()
3301 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase1()
3308 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3322 if (adev->in_s0ix) in amdgpu_device_ip_suspend_phase2()
3325 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase2()
3326 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase2()
3329 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase2()
3333 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_suspend_phase2()
3334 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3339 if (adev->gmc.xgmi.pending_reset && in amdgpu_device_ip_suspend_phase2()
3340 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_suspend_phase2()
3341 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC || in amdgpu_device_ip_suspend_phase2()
3342 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_suspend_phase2()
3343 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) { in amdgpu_device_ip_suspend_phase2()
3344 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3348 /* skip suspend of gfx/mes and psp for S0ix in amdgpu_device_ip_suspend_phase2()
3349 * gfx is in gfxoff state, so on resume it will exit gfxoff just in amdgpu_device_ip_suspend_phase2()
3353 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3354 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || in amdgpu_device_ip_suspend_phase2()
3355 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_ip_suspend_phase2()
3356 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) in amdgpu_device_ip_suspend_phase2()
3359 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */ in amdgpu_device_ip_suspend_phase2()
3360 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3363 (adev->ip_blocks[i].version->type == in amdgpu_device_ip_suspend_phase2()
3367 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. in amdgpu_device_ip_suspend_phase2()
3368 * These are in TMR, hence are expected to be reused by PSP-TOS to reload in amdgpu_device_ip_suspend_phase2()
3370 * from here based on PMFW -> PSP message during re-init sequence. in amdgpu_device_ip_suspend_phase2()
3375 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && in amdgpu_device_ip_suspend_phase2()
3376 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_suspend_phase2()
3380 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase2()
3384 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase2()
3386 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3389 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_suspend_phase2()
3390 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); in amdgpu_device_ip_suspend_phase2()
3393 adev->mp1_state, r); in amdgpu_device_ip_suspend_phase2()
3404 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3447 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_reinit_early_sriov()
3451 block = &adev->ip_blocks[i]; in amdgpu_device_ip_reinit_early_sriov()
3452 block->status.hw = false; in amdgpu_device_ip_reinit_early_sriov()
3456 if (block->version->type != ip_order[j] || in amdgpu_device_ip_reinit_early_sriov()
3457 !block->status.valid) in amdgpu_device_ip_reinit_early_sriov()
3460 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_early_sriov()
3461 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); in amdgpu_device_ip_reinit_early_sriov()
3464 block->status.hw = true; in amdgpu_device_ip_reinit_early_sriov()
3491 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_late_sriov()
3492 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_late_sriov()
3494 if (block->version->type != ip_order[i] || in amdgpu_device_ip_reinit_late_sriov()
3495 !block->status.valid || in amdgpu_device_ip_reinit_late_sriov()
3496 block->status.hw) in amdgpu_device_ip_reinit_late_sriov()
3499 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) in amdgpu_device_ip_reinit_late_sriov()
3500 r = block->version->funcs->resume(adev); in amdgpu_device_ip_reinit_late_sriov()
3502 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_late_sriov()
3504 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); in amdgpu_device_ip_reinit_late_sriov()
3507 block->status.hw = true; in amdgpu_device_ip_reinit_late_sriov()
3515 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3530 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase1()
3531 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase1()
3533 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase1()
3534 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase1()
3535 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase1()
3536 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { in amdgpu_device_ip_resume_phase1()
3538 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase1()
3541 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase1()
3544 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase1()
3552 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3568 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase2()
3569 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase2()
3571 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase2()
3572 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase2()
3573 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase2()
3574 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_resume_phase2()
3576 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase2()
3579 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase2()
3582 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase2()
3589 * amdgpu_device_ip_resume - run resume for hardware IPs
3614 if (adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_device_ip_resume()
3621 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3625 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3630 if (adev->is_atom_fw) { in amdgpu_device_detect_sriov_bios()
3632 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3635 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3638 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) in amdgpu_device_detect_sriov_bios()
3644 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3669 * Fallback to the non-DC driver here by default so as not to in amdgpu_device_asic_has_dc_support()
3685 * Fallback to the non-DC driver here by default so as not to in amdgpu_device_asic_has_dc_support()
3701 * amdgpu_device_has_dc_support - check if dc is supported
3709 if (adev->enable_virtual_display || in amdgpu_device_has_dc_support()
3710 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_dc_support()
3713 return amdgpu_device_asic_has_dc_support(adev->asic_type); in amdgpu_device_has_dc_support()
3734 task_barrier_enter(&hive->tb); in amdgpu_device_xgmi_reset_func()
3735 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); in amdgpu_device_xgmi_reset_func()
3737 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3740 task_barrier_exit(&hive->tb); in amdgpu_device_xgmi_reset_func()
3741 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); in amdgpu_device_xgmi_reset_func()
3743 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3749 task_barrier_full(&hive->tb); in amdgpu_device_xgmi_reset_func()
3750 adev->asic_reset_res = amdgpu_asic_reset(adev); in amdgpu_device_xgmi_reset_func()
3754 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3756 adev->asic_reset_res, adev_to_drm(adev)->unique); in amdgpu_device_xgmi_reset_func()
3771 * In SR-IOV or passthrough mode, timeout for compute in amdgpu_device_get_job_timeout_settings()
3774 adev->gfx_timeout = msecs_to_jiffies(10000); in amdgpu_device_get_job_timeout_settings()
3775 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3777 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? in amdgpu_device_get_job_timeout_settings()
3780 adev->compute_timeout = msecs_to_jiffies(60000); in amdgpu_device_get_job_timeout_settings()
3794 dev_warn(adev->dev, "lockup timeout disabled"); in amdgpu_device_get_job_timeout_settings()
3802 adev->gfx_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3805 adev->compute_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3808 adev->sdma_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3811 adev->video_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3819 * it should apply to all non-compute jobs. in amdgpu_device_get_job_timeout_settings()
3822 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3824 adev->compute_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3832 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3842 domain = iommu_get_domain_for_dev(adev->dev); in amdgpu_device_check_iommu_direct_map()
3843 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) in amdgpu_device_check_iommu_direct_map()
3844 adev->ram_is_direct_mapped = true; in amdgpu_device_check_iommu_direct_map()
3855 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3857 adev->gfx.mcbp = false; in amdgpu_device_set_mcbp()
3860 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3862 if (adev->gfx.mcbp) in amdgpu_device_set_mcbp()
3867 * amdgpu_device_init - initialize the driver
3880 struct pci_dev *pdev = adev->pdev; in amdgpu_device_init()
3886 adev->shutdown = false; in amdgpu_device_init()
3887 adev->flags = flags; in amdgpu_device_init()
3890 adev->asic_type = amdgpu_force_asic_type; in amdgpu_device_init()
3892 adev->asic_type = flags & AMD_ASIC_MASK; in amdgpu_device_init()
3894 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; in amdgpu_device_init()
3896 adev->usec_timeout *= 10; in amdgpu_device_init()
3897 adev->gmc.gart_size = 512 * 1024 * 1024; in amdgpu_device_init()
3898 adev->accel_working = false; in amdgpu_device_init()
3899 adev->num_rings = 0; in amdgpu_device_init()
3900 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); in amdgpu_device_init()
3901 adev->mman.buffer_funcs = NULL; in amdgpu_device_init()
3902 adev->mman.buffer_funcs_ring = NULL; in amdgpu_device_init()
3903 adev->vm_manager.vm_pte_funcs = NULL; in amdgpu_device_init()
3904 adev->vm_manager.vm_pte_num_scheds = 0; in amdgpu_device_init()
3905 adev->gmc.gmc_funcs = NULL; in amdgpu_device_init()
3906 adev->harvest_ip_mask = 0x0; in amdgpu_device_init()
3907 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); in amdgpu_device_init()
3908 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in amdgpu_device_init()
3910 adev->smc_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3911 adev->smc_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3912 adev->pcie_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3913 adev->pcie_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3914 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext; in amdgpu_device_init()
3915 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext; in amdgpu_device_init()
3916 adev->pciep_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3917 adev->pciep_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3918 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; in amdgpu_device_init()
3919 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; in amdgpu_device_init()
3920 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext; in amdgpu_device_init()
3921 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext; in amdgpu_device_init()
3922 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3923 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3924 adev->didt_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3925 adev->didt_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3926 adev->gc_cac_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
3927 adev->gc_cac_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
3928 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; in amdgpu_device_init()
3929 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; in amdgpu_device_init()
3932 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, in amdgpu_device_init()
3933 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); in amdgpu_device_init()
3938 mutex_init(&adev->firmware.mutex); in amdgpu_device_init()
3939 mutex_init(&adev->pm.mutex); in amdgpu_device_init()
3940 mutex_init(&adev->gfx.gpu_clock_mutex); in amdgpu_device_init()
3941 mutex_init(&adev->srbm_mutex); in amdgpu_device_init()
3942 mutex_init(&adev->gfx.pipe_reserve_mutex); in amdgpu_device_init()
3943 mutex_init(&adev->gfx.gfx_off_mutex); in amdgpu_device_init()
3944 mutex_init(&adev->gfx.partition_mutex); in amdgpu_device_init()
3945 mutex_init(&adev->grbm_idx_mutex); in amdgpu_device_init()
3946 mutex_init(&adev->mn_lock); in amdgpu_device_init()
3947 mutex_init(&adev->virt.vf_errors.lock); in amdgpu_device_init()
3948 hash_init(adev->mn_hash); in amdgpu_device_init()
3949 mutex_init(&adev->psp.mutex); in amdgpu_device_init()
3950 mutex_init(&adev->notifier_lock); in amdgpu_device_init()
3951 mutex_init(&adev->pm.stable_pstate_ctx_lock); in amdgpu_device_init()
3952 mutex_init(&adev->benchmark_mutex); in amdgpu_device_init()
3960 spin_lock_init(&adev->mmio_idx_lock); in amdgpu_device_init()
3961 spin_lock_init(&adev->smc_idx_lock); in amdgpu_device_init()
3962 spin_lock_init(&adev->pcie_idx_lock); in amdgpu_device_init()
3963 spin_lock_init(&adev->uvd_ctx_idx_lock); in amdgpu_device_init()
3964 spin_lock_init(&adev->didt_idx_lock); in amdgpu_device_init()
3965 spin_lock_init(&adev->gc_cac_idx_lock); in amdgpu_device_init()
3966 spin_lock_init(&adev->se_cac_idx_lock); in amdgpu_device_init()
3967 spin_lock_init(&adev->audio_endpt_idx_lock); in amdgpu_device_init()
3968 spin_lock_init(&adev->mm_stats.lock); in amdgpu_device_init()
3970 INIT_LIST_HEAD(&adev->shadow_list); in amdgpu_device_init()
3971 mutex_init(&adev->shadow_list_lock); in amdgpu_device_init()
3973 INIT_LIST_HEAD(&adev->reset_list); in amdgpu_device_init()
3975 INIT_LIST_HEAD(&adev->ras_list); in amdgpu_device_init()
3977 INIT_LIST_HEAD(&adev->pm.od_kobj_list); in amdgpu_device_init()
3979 INIT_DELAYED_WORK(&adev->delayed_init_work, in amdgpu_device_init()
3981 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, in amdgpu_device_init()
3984 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); in amdgpu_device_init()
3986 adev->gfx.gfx_off_req_count = 1; in amdgpu_device_init()
3987 adev->gfx.gfx_off_residency = 0; in amdgpu_device_init()
3988 adev->gfx.gfx_off_entrycount = 0; in amdgpu_device_init()
3989 adev->pm.ac_power = power_supply_is_system_supplied() > 0; in amdgpu_device_init()
3991 atomic_set(&adev->throttling_logging_enabled, 1); in amdgpu_device_init()
3994 * to avoid log flooding. "-1" is subtracted since the thermal in amdgpu_device_init()
3999 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); in amdgpu_device_init()
4000 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); in amdgpu_device_init()
4004 if (adev->asic_type >= CHIP_BONAIRE) { in amdgpu_device_init()
4005 adev->rmmio_base = pci_resource_start(adev->pdev, 5); in amdgpu_device_init()
4006 adev->rmmio_size = pci_resource_len(adev->pdev, 5); in amdgpu_device_init()
4008 adev->rmmio_base = pci_resource_start(adev->pdev, 2); in amdgpu_device_init()
4009 adev->rmmio_size = pci_resource_len(adev->pdev, 2); in amdgpu_device_init()
4013 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); in amdgpu_device_init()
4015 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); in amdgpu_device_init()
4016 if (!adev->rmmio) in amdgpu_device_init()
4017 return -ENOMEM; in amdgpu_device_init()
4019 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); in amdgpu_device_init()
4020 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size); in amdgpu_device_init()
4027 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); in amdgpu_device_init()
4028 if (!adev->reset_domain) in amdgpu_device_init()
4029 return -ENOMEM; in amdgpu_device_init()
4038 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); in amdgpu_device_init()
4050 r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver); in amdgpu_device_init()
4059 if (adev->gmc.xgmi.supported) { in amdgpu_device_init()
4060 r = adev->gfxhub.funcs->get_xgmi_info(adev); in amdgpu_device_init()
4067 if (adev->virt.fw_reserve.p_pf2vf) in amdgpu_device_init()
4068 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) in amdgpu_device_init()
4069 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == in amdgpu_device_init()
4074 } else if ((adev->flags & AMD_IS_APU) && in amdgpu_device_init()
4077 adev->have_atomics_support = true; in amdgpu_device_init()
4079 adev->have_atomics_support = in amdgpu_device_init()
4080 !pci_enable_atomic_ops_to_root(adev->pdev, in amdgpu_device_init()
4085 if (!adev->have_atomics_support) in amdgpu_device_init()
4086 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); in amdgpu_device_init()
4100 if (adev->bios) in amdgpu_device_init()
4107 if (adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_init()
4108 dev_info(adev->dev, "Pending hive reset.\n"); in amdgpu_device_init()
4109 adev->gmc.xgmi.pending_reset = true; in amdgpu_device_init()
4111 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_init()
4112 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_init()
4114 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_init()
4115 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_init()
4116 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_init()
4117 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) { in amdgpu_device_init()
4119 adev->ip_blocks[i].version->funcs->name); in amdgpu_device_init()
4120 adev->ip_blocks[i].status.hw = true; in amdgpu_device_init()
4132 dev_err(adev->dev, "asic reset on init failed\n"); in amdgpu_device_init()
4140 if (!adev->bios) { in amdgpu_device_init()
4141 dev_err(adev->dev, "no vBIOS found\n"); in amdgpu_device_init()
4142 r = -EINVAL; in amdgpu_device_init()
4148 dev_err(adev->dev, "gpu post error!\n"); in amdgpu_device_init()
4153 if (adev->bios) { in amdgpu_device_init()
4154 if (adev->is_atom_fw) { in amdgpu_device_init()
4158 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); in amdgpu_device_init()
4166 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); in amdgpu_device_init()
4180 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); in amdgpu_device_init()
4190 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); in amdgpu_device_init()
4197 dev_info(adev->dev, in amdgpu_device_init()
4199 adev->gfx.config.max_shader_engines, in amdgpu_device_init()
4200 adev->gfx.config.max_sh_per_se, in amdgpu_device_init()
4201 adev->gfx.config.max_cu_per_sh, in amdgpu_device_init()
4202 adev->gfx.cu_info.number); in amdgpu_device_init()
4204 adev->accel_working = true; in amdgpu_device_init()
4214 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); in amdgpu_device_init()
4226 if (!adev->gmc.xgmi.pending_reset) { in amdgpu_device_init()
4229 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); in amdgpu_device_init()
4235 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_init()
4241 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_init()
4251 drm_err(&adev->ddev, in amdgpu_device_init()
4260 adev->ucode_sysfs_en = false; in amdgpu_device_init()
4263 adev->ucode_sysfs_en = true; in amdgpu_device_init()
4265 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); in amdgpu_device_init()
4267 dev_err(adev->dev, "Could not create amdgpu device attr\n"); in amdgpu_device_init()
4269 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); in amdgpu_device_init()
4271 dev_err(adev->dev, in amdgpu_device_init()
4280 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); in amdgpu_device_init()
4283 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_device_init()
4290 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_init()
4291 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); in amdgpu_device_init()
4295 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_init()
4297 vga_switcheroo_register_client(adev->pdev, in amdgpu_device_init()
4301 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); in amdgpu_device_init()
4303 if (adev->gmc.xgmi.pending_reset) in amdgpu_device_init()
4320 dev_err(adev->dev, "VF exclusive mode timeout\n"); in amdgpu_device_init()
4322 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; in amdgpu_device_init()
4323 adev->virt.ops = NULL; in amdgpu_device_init()
4324 r = -EAGAIN; in amdgpu_device_init()
4338 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); in amdgpu_device_unmap_mmio()
4340 /* Unmap all mapped bars - Doorbell, registers and VRAM */ in amdgpu_device_unmap_mmio()
4343 iounmap(adev->rmmio); in amdgpu_device_unmap_mmio()
4344 adev->rmmio = NULL; in amdgpu_device_unmap_mmio()
4345 if (adev->mman.aper_base_kaddr) in amdgpu_device_unmap_mmio()
4346 iounmap(adev->mman.aper_base_kaddr); in amdgpu_device_unmap_mmio()
4347 adev->mman.aper_base_kaddr = NULL; in amdgpu_device_unmap_mmio()
4350 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { in amdgpu_device_unmap_mmio()
4351 arch_phys_wc_del(adev->gmc.vram_mtrr); in amdgpu_device_unmap_mmio()
4352 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); in amdgpu_device_unmap_mmio()
4357 * amdgpu_device_fini_hw - tear down the driver
4366 dev_info(adev->dev, "amdgpu: finishing device.\n"); in amdgpu_device_fini_hw()
4367 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_fini_hw()
4368 adev->shutdown = true; in amdgpu_device_fini_hw()
4380 if (adev->mode_info.mode_config_initialized) { in amdgpu_device_fini_hw()
4388 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4389 drain_workqueue(adev->mman.bdev.wq); in amdgpu_device_fini_hw()
4391 if (adev->pm.sysfs_initialized) in amdgpu_device_fini_hw()
4393 if (adev->ucode_sysfs_en) in amdgpu_device_fini_hw()
4395 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); in amdgpu_device_fini_hw()
4409 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4410 ttm_device_clear_dma_mappings(&adev->mman.bdev); in amdgpu_device_fini_hw()
4426 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); in amdgpu_device_fini_sw()
4427 adev->accel_working = false; in amdgpu_device_fini_sw()
4428 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); in amdgpu_device_fini_sw()
4439 kfree(adev->bios); in amdgpu_device_fini_sw()
4440 adev->bios = NULL; in amdgpu_device_fini_sw()
4442 kfree(adev->fru_info); in amdgpu_device_fini_sw()
4443 adev->fru_info = NULL; in amdgpu_device_fini_sw()
4447 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_fini_sw()
4449 vga_switcheroo_unregister_client(adev->pdev); in amdgpu_device_fini_sw()
4452 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_fini_sw()
4454 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_fini_sw()
4455 vga_client_unregister(adev->pdev); in amdgpu_device_fini_sw()
4459 iounmap(adev->rmmio); in amdgpu_device_fini_sw()
4460 adev->rmmio = NULL; in amdgpu_device_fini_sw()
4467 if (adev->mman.discovery_bin) in amdgpu_device_fini_sw()
4470 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_fini_sw()
4471 adev->reset_domain = NULL; in amdgpu_device_fini_sw()
4473 kfree(adev->pci_state); in amdgpu_device_fini_sw()
4478 * amdgpu_device_evict_resources - evict device resources
4491 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) in amdgpu_device_evict_resources()
4504 * amdgpu_device_prepare - prepare for device suspend
4519 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_prepare()
4527 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_prepare()
4528 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_prepare()
4530 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) in amdgpu_device_prepare()
4532 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev); in amdgpu_device_prepare()
4540 adev->in_s0ix = adev->in_s3 = false; in amdgpu_device_prepare()
4546 * amdgpu_device_suspend - initiate device suspend
4560 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_suspend()
4563 adev->in_suspend = true; in amdgpu_device_suspend()
4576 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true); in amdgpu_device_suspend()
4578 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_suspend()
4584 if (!adev->in_s0ix) in amdgpu_device_suspend()
4585 amdgpu_amdkfd_suspend(adev, adev->in_runpm); in amdgpu_device_suspend()
4608 * amdgpu_device_resume - initiate device resume
4628 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_resume()
4631 if (adev->in_s0ix) in amdgpu_device_resume()
4638 dev_err(adev->dev, "amdgpu asic init failed\n"); in amdgpu_device_resume()
4644 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); in amdgpu_device_resume()
4649 if (!adev->in_s0ix) { in amdgpu_device_resume()
4650 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); in amdgpu_device_resume()
4659 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_resume()
4671 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_resume()
4674 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); in amdgpu_device_resume()
4678 if (adev->mode_info.num_crtc) { in amdgpu_device_resume()
4689 dev->dev->power.disable_depth++; in amdgpu_device_resume()
4691 if (!adev->dc_enabled) in amdgpu_device_resume()
4696 dev->dev->power.disable_depth--; in amdgpu_device_resume()
4699 adev->in_suspend = false; in amdgpu_device_resume()
4701 if (adev->enable_mes) in amdgpu_device_resume()
4711 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4731 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_check_soft_reset()
4732 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_check_soft_reset()
4734 if (adev->ip_blocks[i].version->funcs->check_soft_reset) in amdgpu_device_ip_check_soft_reset()
4735 adev->ip_blocks[i].status.hang = in amdgpu_device_ip_check_soft_reset()
4736 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); in amdgpu_device_ip_check_soft_reset()
4737 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_check_soft_reset()
4738 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_check_soft_reset()
4746 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4760 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_pre_soft_reset()
4761 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_pre_soft_reset()
4763 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_pre_soft_reset()
4764 adev->ip_blocks[i].version->funcs->pre_soft_reset) { in amdgpu_device_ip_pre_soft_reset()
4765 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); in amdgpu_device_ip_pre_soft_reset()
4775 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4790 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_need_full_reset()
4791 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_need_full_reset()
4793 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || in amdgpu_device_ip_need_full_reset()
4794 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || in amdgpu_device_ip_need_full_reset()
4795 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || in amdgpu_device_ip_need_full_reset()
4796 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || in amdgpu_device_ip_need_full_reset()
4797 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_need_full_reset()
4798 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_need_full_reset()
4799 dev_info(adev->dev, "Some block need full reset!\n"); in amdgpu_device_ip_need_full_reset()
4808 * amdgpu_device_ip_soft_reset - do a soft reset
4822 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_soft_reset()
4823 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_soft_reset()
4825 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_soft_reset()
4826 adev->ip_blocks[i].version->funcs->soft_reset) { in amdgpu_device_ip_soft_reset()
4827 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); in amdgpu_device_ip_soft_reset()
4837 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4851 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_post_soft_reset()
4852 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_post_soft_reset()
4854 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_post_soft_reset()
4855 adev->ip_blocks[i].version->funcs->post_soft_reset) in amdgpu_device_ip_post_soft_reset()
4856 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); in amdgpu_device_ip_post_soft_reset()
4865 * amdgpu_device_recover_vram - Recover some VRAM contents
4888 dev_info(adev->dev, "recover vram bo from shadow start\n"); in amdgpu_device_recover_vram()
4889 mutex_lock(&adev->shadow_list_lock); in amdgpu_device_recover_vram()
4890 list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) { in amdgpu_device_recover_vram()
4892 if (!vmbo->shadow) in amdgpu_device_recover_vram()
4894 shadow = vmbo->shadow; in amdgpu_device_recover_vram()
4897 if (shadow->tbo.resource->mem_type != TTM_PL_TT || in amdgpu_device_recover_vram()
4898 shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET || in amdgpu_device_recover_vram()
4899 shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM) in amdgpu_device_recover_vram()
4911 r = -ETIMEDOUT; in amdgpu_device_recover_vram()
4921 mutex_unlock(&adev->shadow_list_lock); in amdgpu_device_recover_vram()
4928 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); in amdgpu_device_recover_vram()
4929 return -EIO; in amdgpu_device_recover_vram()
4932 dev_info(adev->dev, "recover vram bo from shadow done\n"); in amdgpu_device_recover_vram()
4938 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4985 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_reset_sriov()
4998 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { in amdgpu_device_reset_sriov()
5016 * amdgpu_device_has_job_running - check if there is any job in mirror list
5028 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_has_job_running()
5033 spin_lock(&ring->sched.job_list_lock); in amdgpu_device_has_job_running()
5034 job = list_first_entry_or_null(&ring->sched.pending_list, in amdgpu_device_has_job_running()
5036 spin_unlock(&ring->sched.job_list_lock); in amdgpu_device_has_job_running()
5044 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5064 if (amdgpu_gpu_recovery == -1) { in amdgpu_device_should_recover_gpu()
5065 switch (adev->asic_type) { in amdgpu_device_should_recover_gpu()
5090 dev_info(adev->dev, "GPU recovery disabled.\n"); in amdgpu_device_should_recover_gpu()
5101 dev_info(adev->dev, "GPU mode1 reset\n"); in amdgpu_device_mode1_reset()
5104 pci_clear_master(adev->pdev); in amdgpu_device_mode1_reset()
5106 amdgpu_device_cache_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5109 dev_info(adev->dev, "GPU smu mode1 reset\n"); in amdgpu_device_mode1_reset()
5112 dev_info(adev->dev, "GPU psp mode1 reset\n"); in amdgpu_device_mode1_reset()
5119 amdgpu_device_load_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5125 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_device_mode1_reset()
5126 u32 memsize = adev->nbio.funcs->get_memsize(adev); in amdgpu_device_mode1_reset()
5133 if (i >= adev->usec_timeout) { in amdgpu_device_mode1_reset()
5134 ret = -ETIMEDOUT; in amdgpu_device_mode1_reset()
5143 dev_err(adev->dev, "GPU mode1 reset failed\n"); in amdgpu_device_mode1_reset()
5153 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5155 if (reset_context->reset_req_dev == adev) in amdgpu_device_pre_asic_reset()
5156 job = reset_context->job; in amdgpu_device_pre_asic_reset()
5167 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_pre_asic_reset()
5183 if (job && job->vm) in amdgpu_device_pre_asic_reset()
5184 drm_sched_increase_karma(&job->base); in amdgpu_device_pre_asic_reset()
5188 if (r == -EOPNOTSUPP) in amdgpu_device_pre_asic_reset()
5205 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); in amdgpu_device_pre_asic_reset()
5213 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5216 &reset_context->flags); in amdgpu_device_pre_asic_reset()
5226 lockdep_assert_held(&adev->reset_domain->sem); in amdgpu_reset_reg_dumps()
5228 for (i = 0; i < adev->reset_info.num_regs; i++) { in amdgpu_reset_reg_dumps()
5229 adev->reset_info.reset_dump_reg_value[i] = in amdgpu_reset_reg_dumps()
5230 RREG32(adev->reset_info.reset_dump_reg_list[i]); in amdgpu_reset_reg_dumps()
5232 trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i], in amdgpu_reset_reg_dumps()
5233 adev->reset_info.reset_dump_reg_value[i]); in amdgpu_reset_reg_dumps()
5251 reset_context->reset_device_list = device_list_handle; in amdgpu_do_asic_reset()
5254 if (r == -EOPNOTSUPP) in amdgpu_do_asic_reset()
5261 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5262 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5271 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5272 tmp_adev->gmc.xgmi.pending_reset = false; in amdgpu_do_asic_reset()
5273 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work)) in amdgpu_do_asic_reset()
5274 r = -EALREADY; in amdgpu_do_asic_reset()
5279 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", in amdgpu_do_asic_reset()
5280 r, adev_to_drm(tmp_adev)->unique); in amdgpu_do_asic_reset()
5288 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5289 flush_work(&tmp_adev->xgmi_reset_work); in amdgpu_do_asic_reset()
5290 r = tmp_adev->asic_reset_res; in amdgpu_do_asic_reset()
5311 dev_warn(tmp_adev->dev, "asic atom init failed!"); in amdgpu_do_asic_reset()
5313 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); in amdgpu_do_asic_reset()
5333 tmp_adev->xcp_mgr); in amdgpu_do_asic_reset()
5341 if (tmp_adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_do_asic_reset()
5353 if (!reset_context->hive && in amdgpu_do_asic_reset()
5354 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_do_asic_reset()
5361 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false); in amdgpu_do_asic_reset()
5377 r = -EINVAL; in amdgpu_do_asic_reset()
5382 if (reset_context->hive && in amdgpu_do_asic_reset()
5383 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_do_asic_reset()
5385 reset_context->hive, tmp_adev); in amdgpu_do_asic_reset()
5394 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); in amdgpu_do_asic_reset()
5396 r = -EAGAIN; in amdgpu_do_asic_reset()
5404 tmp_adev->asic_reset_res = r; in amdgpu_do_asic_reset()
5409 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5411 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5420 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; in amdgpu_device_set_mp1_state()
5423 adev->mp1_state = PP_MP1_STATE_RESET; in amdgpu_device_set_mp1_state()
5426 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_set_mp1_state()
5434 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_unset_mp1_state()
5441 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_resume_display_audio()
5442 adev->pdev->bus->number, 1); in amdgpu_device_resume_display_audio()
5444 pm_runtime_enable(&(p->dev)); in amdgpu_device_resume_display_audio()
5445 pm_runtime_resume(&(p->dev)); in amdgpu_device_resume_display_audio()
5464 return -EINVAL; in amdgpu_device_suspend_display_audio()
5466 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_suspend_display_audio()
5467 adev->pdev->bus->number, 1); in amdgpu_device_suspend_display_audio()
5469 return -ENODEV; in amdgpu_device_suspend_display_audio()
5471 expires = pm_runtime_autosuspend_expiration(&(p->dev)); in amdgpu_device_suspend_display_audio()
5481 while (!pm_runtime_status_suspended(&(p->dev))) { in amdgpu_device_suspend_display_audio()
5482 if (!pm_runtime_suspend(&(p->dev))) in amdgpu_device_suspend_display_audio()
5486 dev_warn(adev->dev, "failed to suspend display audio\n"); in amdgpu_device_suspend_display_audio()
5489 return -ETIMEDOUT; in amdgpu_device_suspend_display_audio()
5493 pm_runtime_disable(&(p->dev)); in amdgpu_device_suspend_display_audio()
5505 cancel_work(&adev->reset_work); in amdgpu_device_stop_pending_resets()
5508 if (adev->kfd.dev) in amdgpu_device_stop_pending_resets()
5509 cancel_work(&adev->kfd.reset_work); in amdgpu_device_stop_pending_resets()
5512 cancel_work(&adev->virt.flr_work); in amdgpu_device_stop_pending_resets()
5514 if (con && adev->ras_enabled) in amdgpu_device_stop_pending_resets()
5515 cancel_work(&con->recovery_work); in amdgpu_device_stop_pending_resets()
5520 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5527 * Attempt to do soft-reset or full-reset and reinitialize Asic
5553 amdgpu_ras_get_context(adev)->reboot) { in amdgpu_device_gpu_recover()
5560 dev_info(adev->dev, "GPU %s begin!\n", in amdgpu_device_gpu_recover()
5566 mutex_lock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5568 reset_context->job = job; in amdgpu_device_gpu_recover()
5569 reset_context->hive = hive; in amdgpu_device_gpu_recover()
5576 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) { in amdgpu_device_gpu_recover()
5577 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { in amdgpu_device_gpu_recover()
5578 list_add_tail(&tmp_adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5579 if (adev->shutdown) in amdgpu_device_gpu_recover()
5580 tmp_adev->shutdown = true; in amdgpu_device_gpu_recover()
5582 if (!list_is_first(&adev->reset_list, &device_list)) in amdgpu_device_gpu_recover()
5583 list_rotate_to_front(&adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5586 list_add_tail(&adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5593 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_gpu_recover()
5615 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); in amdgpu_device_gpu_recover()
5626 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true); in amdgpu_device_gpu_recover()
5634 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_gpu_recover()
5639 drm_sched_stop(&ring->sched, job ? &job->base : NULL); in amdgpu_device_gpu_recover()
5642 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); in amdgpu_device_gpu_recover()
5644 atomic_inc(&tmp_adev->gpu_reset_counter); in amdgpu_device_gpu_recover()
5654 * job->base holds a reference to parent fence in amdgpu_device_gpu_recover()
5656 if (job && dma_fence_is_signaled(&job->hw_fence)) { in amdgpu_device_gpu_recover()
5658 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); in amdgpu_device_gpu_recover()
5667 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", in amdgpu_device_gpu_recover()
5668 r, adev_to_drm(tmp_adev)->unique); in amdgpu_device_gpu_recover()
5669 tmp_adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
5684 adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
5693 if (r && r == -EAGAIN) in amdgpu_device_gpu_recover()
5703 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_gpu_recover()
5708 drm_sched_start(&ring->sched, true); in amdgpu_device_gpu_recover()
5714 if (tmp_adev->asic_reset_res) in amdgpu_device_gpu_recover()
5715 r = tmp_adev->asic_reset_res; in amdgpu_device_gpu_recover()
5717 tmp_adev->asic_reset_res = 0; in amdgpu_device_gpu_recover()
5721 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
5724 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
5739 if (!adev->kfd.init_complete) in amdgpu_device_gpu_recover()
5752 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_gpu_recover()
5755 mutex_unlock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5760 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); in amdgpu_device_gpu_recover()
5762 atomic_set(&adev->reset_domain->reset_res, r); in amdgpu_device_gpu_recover()
5767 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5781 struct pci_dev *parent = adev->pdev; in amdgpu_device_partner_bandwidth()
5791 if (parent->vendor == PCI_VENDOR_ID_ATI) in amdgpu_device_partner_bandwidth()
5800 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5815 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; in amdgpu_device_get_pcie_info()
5818 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; in amdgpu_device_get_pcie_info()
5821 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { in amdgpu_device_get_pcie_info()
5822 if (adev->pm.pcie_gen_mask == 0) in amdgpu_device_get_pcie_info()
5823 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; in amdgpu_device_get_pcie_info()
5824 if (adev->pm.pcie_mlw_mask == 0) in amdgpu_device_get_pcie_info()
5825 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
5829 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) in amdgpu_device_get_pcie_info()
5835 if (adev->pm.pcie_gen_mask == 0) { in amdgpu_device_get_pcie_info()
5837 pdev = adev->pdev; in amdgpu_device_get_pcie_info()
5840 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5845 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5851 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5856 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5860 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5863 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
5867 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5871 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5877 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5882 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5886 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5889 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
5893 if (adev->pm.pcie_mlw_mask == 0) { in amdgpu_device_get_pcie_info()
5895 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
5899 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
5908 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
5916 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
5923 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
5929 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
5934 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
5938 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()
5948 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5961 uint64_t address_mask = peer_adev->dev->dma_mask ? in amdgpu_device_is_peer_accessible()
5962 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); in amdgpu_device_is_peer_accessible()
5964 adev->gmc.aper_base + adev->gmc.aper_size - 1; in amdgpu_device_is_peer_accessible()
5966 !adev->gmc.xgmi.connected_to_cpu && in amdgpu_device_is_peer_accessible()
5967 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); in amdgpu_device_is_peer_accessible()
5969 return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size && in amdgpu_device_is_peer_accessible()
5970 adev->gmc.real_vram_size == adev->gmc.visible_vram_size && in amdgpu_device_is_peer_accessible()
5971 !(adev->gmc.aper_base & address_mask || in amdgpu_device_is_peer_accessible()
5984 return -ENOTSUPP; in amdgpu_device_baco_enter()
5986 if (ras && adev->ras_enabled && in amdgpu_device_baco_enter()
5987 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_enter()
5988 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); in amdgpu_device_baco_enter()
6000 return -ENOTSUPP; in amdgpu_device_baco_exit()
6006 if (ras && adev->ras_enabled && in amdgpu_device_baco_exit()
6007 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_exit()
6008 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); in amdgpu_device_baco_exit()
6011 adev->nbio.funcs->clear_doorbell_interrupt) in amdgpu_device_baco_exit()
6012 adev->nbio.funcs->clear_doorbell_interrupt(adev); in amdgpu_device_baco_exit()
6018 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6034 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_pci_error_detected()
6039 adev->pci_channel_state = state; in amdgpu_pci_error_detected()
6047 * Locking adev->reset_domain->sem will prevent any external access in amdgpu_pci_error_detected()
6050 amdgpu_device_lock_reset_domain(adev->reset_domain); in amdgpu_pci_error_detected()
6058 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_pci_error_detected()
6063 drm_sched_stop(&ring->sched, NULL); in amdgpu_pci_error_detected()
6065 atomic_inc(&adev->gpu_reset_counter); in amdgpu_pci_error_detected()
6076 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6084 /* TODO - dump whatever for debugging purposes */ in amdgpu_pci_mmio_enabled()
6095 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6116 list_add_tail(&adev->reset_list, &device_list); in amdgpu_pci_slot_reset()
6125 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_pci_slot_reset()
6133 r = -ETIME; in amdgpu_pci_slot_reset()
6142 adev->no_hw_access = true; in amdgpu_pci_slot_reset()
6144 adev->no_hw_access = false; in amdgpu_pci_slot_reset()
6152 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_pci_slot_reset()
6153 pci_restore_state(adev->pdev); in amdgpu_pci_slot_reset()
6159 amdgpu_device_unlock_reset_domain(adev->reset_domain); in amdgpu_pci_slot_reset()
6166 * amdgpu_pci_resume() - resume normal ops after PCI reset
6182 if (adev->pci_channel_state != pci_channel_io_frozen) in amdgpu_pci_resume()
6186 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_pci_resume()
6191 drm_sched_start(&ring->sched, true); in amdgpu_pci_resume()
6195 amdgpu_device_unlock_reset_domain(adev->reset_domain); in amdgpu_pci_resume()
6206 kfree(adev->pci_state); in amdgpu_device_cache_pci_state()
6208 adev->pci_state = pci_store_saved_state(pdev); in amdgpu_device_cache_pci_state()
6210 if (!adev->pci_state) { in amdgpu_device_cache_pci_state()
6228 if (!adev->pci_state) in amdgpu_device_load_pci_state()
6231 r = pci_load_saved_state(pdev, adev->pci_state); in amdgpu_device_load_pci_state()
6247 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_flush_hdp()
6250 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_flush_hdp()
6253 if (ring && ring->funcs->emit_hdp_flush) in amdgpu_device_flush_hdp()
6263 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_invalidate_hdp()
6266 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_invalidate_hdp()
6274 return atomic_read(&adev->reset_domain->in_gpu_reset); in amdgpu_in_reset()
6278 * amdgpu_device_halt() - bring hardware to some kind of halt state
6292 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6299 struct pci_dev *pdev = adev->pdev; in amdgpu_device_halt()
6309 adev->no_hw_access = true; in amdgpu_device_halt()
6323 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); in amdgpu_device_pcie_port_rreg()
6324 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); in amdgpu_device_pcie_port_rreg()
6326 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_rreg()
6330 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_rreg()
6339 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); in amdgpu_device_pcie_port_wreg()
6340 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); in amdgpu_device_pcie_port_wreg()
6342 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_wreg()
6347 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_wreg()
6351 * amdgpu_device_switch_gang - switch to a new gang
6367 old = dma_fence_get_rcu_safe(&adev->gang_submit); in amdgpu_device_switch_gang()
6376 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, in amdgpu_device_switch_gang()
6385 switch (adev->asic_type) { in amdgpu_device_has_display_hardware()
6418 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_display_hardware()
6431 uint32_t loop = adev->usec_timeout; in amdgpu_device_wait_on_rreg()
6435 loop = adev->usec_timeout; in amdgpu_device_wait_on_rreg()
6440 loop--; in amdgpu_device_wait_on_rreg()
6445 ret = -ETIMEDOUT; in amdgpu_device_wait_on_rreg()