190 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,  in amdgpu_ip_member_of_hwini()
196 void amdgpu_set_init_level(struct amdgpu_device *adev,  in amdgpu_set_init_level()
231 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_device_get_pcie_replay_count()  local
240 static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)  in amdgpu_device_attr_sysfs_init()
251 static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)  in amdgpu_device_attr_sysfs_fini()
264 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_sysfs_reg_state_get()  local
298 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)  in amdgpu_reg_state_sysfs_init()
310 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)  in amdgpu_reg_state_sysfs_fini()
338 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_device_get_board_info()  local
372 	struct amdgpu_device *adev = drm_to_adev(ddev);  in amdgpu_board_attrs_is_visible()  local
408 	struct amdgpu_device *adev = drm_to_adev(ddev);  in carveout_options_show()  local
451 	struct amdgpu_device *adev = drm_to_adev(ddev);  in carveout_show()  local
461 	struct amdgpu_device *adev = drm_to_adev(ddev);  in carveout_store()  local
510 static void amdgpu_uma_sysfs_init(struct amdgpu_device *adev)  in amdgpu_uma_sysfs_init()
543 static void amdgpu_uma_sysfs_fini(struct amdgpu_device *adev)  in amdgpu_uma_sysfs_fini()
564 bool amdgpu_device_supports_px(struct amdgpu_device *adev)  in amdgpu_device_supports_px()
579 bool amdgpu_device_supports_boco(struct amdgpu_device *adev)  in amdgpu_device_supports_boco()
600 int amdgpu_device_supports_baco(struct amdgpu_device *adev)  in amdgpu_device_supports_baco()
605 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)  in amdgpu_device_detect_runtime_pm_mode()
689 bool amdgpu_device_supports_smart_shift(struct amdgpu_device *adev)  in amdgpu_device_supports_smart_shift()
708 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_mm_access()
752 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_aper_access()
801 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,  in amdgpu_device_vram_access()
822 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)  in amdgpu_device_skip_hw_access()
858 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,  in amdgpu_device_rreg()
897 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)  in amdgpu_mm_rreg8()
918 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,  in amdgpu_device_xcc_rreg()
965 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)  in amdgpu_mm_wreg8()
986 void amdgpu_device_wreg(struct amdgpu_device *adev,  in amdgpu_device_wreg()
1019 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,  in amdgpu_mm_wreg_mmio_rlc()
1049 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,  in amdgpu_device_xcc_wreg()
1087 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg()
1110 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg_ext()
1170 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg64()
1198 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_rreg64_ext()
1256 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg()
1277 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg_ext()
1325 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg64()
1352 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,  in amdgpu_device_indirect_wreg64_ext()
1408 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)  in amdgpu_device_get_rev_id()
1423 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)  in amdgpu_invalid_rreg()
1430 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)  in amdgpu_invalid_rreg_ext()
1447 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)  in amdgpu_invalid_wreg()
1455 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)  in amdgpu_invalid_wreg_ext()
1473 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)  in amdgpu_invalid_rreg64()
1481 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)  in amdgpu_invalid_rreg64_ext()
1498 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)  in amdgpu_invalid_wreg64()
1506 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)  in amdgpu_invalid_wreg64_ext()
1525 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,  in amdgpu_block_invalid_rreg()
1546 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,  in amdgpu_block_invalid_wreg()
1556 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)  in amdgpu_device_get_vbios_flags()
1574 static int amdgpu_device_asic_init(struct amdgpu_device *adev)  in amdgpu_device_asic_init()
1612 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)  in amdgpu_device_mem_scratch_init()
1629 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)  in amdgpu_device_mem_scratch_fini()
1644 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,  in amdgpu_device_program_register_sequence()
1681 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)  in amdgpu_device_pci_config_reset()
1693 int amdgpu_device_pci_reset(struct amdgpu_device *adev)  in amdgpu_device_pci_reset()
1712 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)  in amdgpu_device_wb_fini()
1731 static int amdgpu_device_wb_init(struct amdgpu_device *adev)  in amdgpu_device_wb_init()
1765 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)  in amdgpu_device_wb_get()
1790 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)  in amdgpu_device_wb_free()
1810 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)  in amdgpu_device_resize_fb_bar()
1909 bool amdgpu_device_need_post(struct amdgpu_device *adev)  in amdgpu_device_need_post()
1973 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)  in amdgpu_device_seamless_boot_supported()
2005 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)  in amdgpu_device_pcie_dynamic_switching_supported()
2020 static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev)  in amdgpu_device_aspm_support_quirk()
2066 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)  in amdgpu_device_should_use_aspm()
2098 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));  in amdgpu_device_vga_set_decode()  local
2118 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)  in amdgpu_device_check_block_size()
2142 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)  in amdgpu_device_check_vm_size()
2155 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)  in amdgpu_device_check_smu_prv_buffer_size()
2195 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)  in amdgpu_device_init_apu_flags()
2240 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)  in amdgpu_device_check_arguments()
2412 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)  in amdgpu_device_enable_virtual_display()
2458 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)  in amdgpu_device_set_sriov_virtual_display()
2479 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)  in amdgpu_device_parse_gpu_info_fw()
2596 static void amdgpu_uid_init(struct amdgpu_device *adev)  in amdgpu_uid_init()
2607 static void amdgpu_uid_fini(struct amdgpu_device *adev)  in amdgpu_uid_fini()
2624 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)  in amdgpu_device_ip_early_init()
2815 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_hw_init_phase1()
2845 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_hw_init_phase2()
2870 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)  in amdgpu_device_fw_loading()
2918 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)  in amdgpu_device_init_schedulers()
2997 static int amdgpu_device_ip_init(struct amdgpu_device *adev)  in amdgpu_device_ip_init()
3188 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)  in amdgpu_device_fill_reset_magic()
3203 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)  in amdgpu_device_check_vram_lost()
3240 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,  in amdgpu_device_set_cg_state()
3281 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,  in amdgpu_device_set_pg_state()
3324 	struct amdgpu_device *adev;  in amdgpu_device_enable_mgpu_fan_boost()  local
3368 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)  in amdgpu_device_ip_late_init()
3454 	struct amdgpu_device *adev = ip_block->adev;  in amdgpu_ip_block_hw_fini()  local
3480 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)  in amdgpu_device_smu_fini_early()
3497 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)  in amdgpu_device_ip_fini_early()
3563 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)  in amdgpu_device_ip_fini()
3629 	struct amdgpu_device *adev =  in amdgpu_device_delayed_init_work_handler()  local
3640 	struct amdgpu_device *adev =  in amdgpu_device_delay_enable_gfx_off()  local
3661 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend_phase1()
3716 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend_phase2()
3838 static int amdgpu_device_ip_suspend(struct amdgpu_device *adev)  in amdgpu_device_ip_suspend()
3860 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)  in amdgpu_device_ip_reinit_early_sriov()
3897 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)  in amdgpu_device_ip_reinit_late_sriov()
3951 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase1()
3985 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase2()
4019 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)  in amdgpu_device_ip_resume_phase3()
4048 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)  in amdgpu_device_ip_resume()
4082 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)  in amdgpu_device_detect_sriov_bios()
4154 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)  in amdgpu_device_has_dc_support()
4165 	struct amdgpu_device *adev =  in amdgpu_device_xgmi_reset_func()  local
4208 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)  in amdgpu_device_get_job_timeout_settings()
4282 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)  in amdgpu_device_check_iommu_direct_map()
4299 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)  in amdgpu_device_check_iommu_remap()
4312 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)  in amdgpu_device_set_mcbp()
4326 static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev)  in amdgpu_device_sys_interface_init()
4363 static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)  in amdgpu_device_sys_interface_fini()
4387 int amdgpu_device_init(struct amdgpu_device *adev,  in amdgpu_device_init()
4872 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)  in amdgpu_device_unmap_mmio()
4902 void amdgpu_device_fini_hw(struct amdgpu_device *adev)  in amdgpu_device_fini_hw()
4963 void amdgpu_device_fini_sw(struct amdgpu_device *adev)  in amdgpu_device_fini_sw()
5037 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)  in amdgpu_device_evict_resources()
5079 	struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);  in amdgpu_device_pm_notifier()  local
5104 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_prepare()  local
5140 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_complete()  local
5164 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_suspend()  local
5269 static inline int amdgpu_virt_resume(struct amdgpu_device *adev)  in amdgpu_virt_resume()
5307 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_resume()  local
5417 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_check_soft_reset()
5454 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_pre_soft_reset()
5481 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_need_full_reset()
5516 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_soft_reset()
5545 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)  in amdgpu_device_ip_post_soft_reset()
5571 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,  in amdgpu_device_reset_sriov()
5656 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)  in amdgpu_device_has_job_running()
5680 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)  in amdgpu_device_should_recover_gpu()
5723 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)  in amdgpu_device_mode1_reset()
5787 int amdgpu_device_link_reset(struct amdgpu_device *adev)  in amdgpu_device_link_reset()
5810 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,  in amdgpu_device_pre_asic_reset()
6111 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)  in amdgpu_device_set_mp1_state()
6128 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)  in amdgpu_device_unset_mp1_state()
6134 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)  in amdgpu_device_resume_display_audio()
6148 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)  in amdgpu_device_suspend_display_audio()
6196 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)  in amdgpu_device_stop_pending_resets()
6229 static void amdgpu_device_recovery_prepare(struct amdgpu_device *adev,  in amdgpu_device_recovery_prepare()
6255 static void amdgpu_device_recovery_get_reset_lock(struct amdgpu_device *adev,  in amdgpu_device_recovery_get_reset_lock()
6267 static void amdgpu_device_recovery_put_reset_lock(struct amdgpu_device *adev,  in amdgpu_device_recovery_put_reset_lock()
6279 static void amdgpu_device_halt_activities(struct amdgpu_device *adev,  in amdgpu_device_halt_activities()
6342 static int amdgpu_device_asic_reset(struct amdgpu_device *adev,  in amdgpu_device_asic_reset()
6455 static void amdgpu_device_gpu_resume(struct amdgpu_device *adev,  in amdgpu_device_gpu_resume()
6495 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,  in amdgpu_device_gpu_recover()
6633 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,  in amdgpu_device_partner_bandwidth()
6670 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,  in amdgpu_device_gpu_bandwidth()
6707 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)  in amdgpu_device_get_pcie_info()
6906 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,  in amdgpu_device_is_peer_accessible()
6936 int amdgpu_device_baco_enter(struct amdgpu_device *adev)  in amdgpu_device_baco_enter()
6950 int amdgpu_device_baco_exit(struct amdgpu_device *adev)  in amdgpu_device_baco_exit()
6985 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_error_detected()  local
7045 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_mmio_enabled()  local
7070 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_slot_reset()  local
7174 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_pci_resume()  local
7207 static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev)  in amdgpu_device_cache_switch_state()
7240 static void amdgpu_device_load_switch_state(struct amdgpu_device *adev)  in amdgpu_device_load_switch_state()
7269 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_cache_pci_state()  local
7298 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_device_load_pci_state()  local
7316 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,  in amdgpu_device_flush_hdp()
7339 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,  in amdgpu_device_invalidate_hdp()
7352 int amdgpu_in_reset(struct amdgpu_device *adev)  in amdgpu_in_reset()
7377 void amdgpu_device_halt(struct amdgpu_device *adev)  in amdgpu_device_halt()
7400 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,  in amdgpu_device_pcie_port_rreg()
7417 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,  in amdgpu_device_pcie_port_wreg()
7439 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)  in amdgpu_device_get_gang()
7458 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,  in amdgpu_device_switch_gang()
7497 struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,  in amdgpu_device_enforce_isolation()
7574 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)  in amdgpu_device_has_display_hardware()
7615 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,  in amdgpu_device_wait_on_rreg()