161 	return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_name);  in amdgpu_device_get_product_name()
183 	return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_number);  in amdgpu_device_get_product_number()
205 	return snprintf(buf, PAGE_SIZE, "%s\n", adev->serial);  in amdgpu_device_get_serial_number()
212  * amdgpu_device_supports_boco - Is the device a dGPU with HG/PX power control
223 	if (adev->flags & AMD_IS_PX)  in amdgpu_device_supports_boco()
229  * amdgpu_device_supports_baco - Does the device support BACO
248  * amdgpu_device_vram_access - read/write a buffer in vram
251  * @pos: offset of the buffer in vram
254  * @write: true - write to vram, otherwise - read from vram
265 	last = min(pos + size, adev->gmc.visible_vram_size);  in amdgpu_device_vram_access()
267 		void __iomem *addr = adev->mman.aper_base_kaddr + pos;  in amdgpu_device_vram_access()
268 		size_t count = last - pos;  in amdgpu_device_vram_access()
285 		size -= count;  in amdgpu_device_vram_access()
289 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);  in amdgpu_device_vram_access()
303 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);  in amdgpu_device_vram_access()
310  * amdgpu_device_rreg - read a memory mapped IO or indirect register
313  * @reg: dword aligned register offset
316  * Returns the 32 bit value from the offset specified.
323 	if (adev->in_pci_err_recovery)  in amdgpu_device_rreg()
326 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_rreg()
329 		    down_read_trylock(&adev->reset_sem)) {  in amdgpu_device_rreg()
331 			up_read(&adev->reset_sem);  in amdgpu_device_rreg()
333 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_rreg()
336 		ret = adev->pcie_rreg(adev, reg * 4);  in amdgpu_device_rreg()
339 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);  in amdgpu_device_rreg()
346  * @offset:bytes offset from MMIO start
351  * amdgpu_mm_rreg8 - read a memory mapped IO register
354  * @offset: byte aligned register offset
356  * Returns the 8 bit value from the offset specified.
358 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)  in amdgpu_mm_rreg8()  argument
360 	if (adev->in_pci_err_recovery)  in amdgpu_mm_rreg8()
363 	if (offset < adev->rmmio_size)  in amdgpu_mm_rreg8()
364 		return (readb(adev->rmmio + offset));  in amdgpu_mm_rreg8()
370  * @offset:bytes offset from MMIO start
375  * amdgpu_mm_wreg8 - read a memory mapped IO register
378  * @offset: byte aligned register offset
381  * Writes the value specified to the offset specified.
383 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)  in amdgpu_mm_wreg8()  argument
385 	if (adev->in_pci_err_recovery)  in amdgpu_mm_wreg8()
388 	if (offset < adev->rmmio_size)  in amdgpu_mm_wreg8()
389 		writeb(value, adev->rmmio + offset);  in amdgpu_mm_wreg8()
395  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
398  * @reg: dword aligned register offset
402  * Writes the value specified to the offset specified.
408 	if (adev->in_pci_err_recovery)  in amdgpu_device_wreg()
411 	if ((reg * 4) < adev->rmmio_size) {  in amdgpu_device_wreg()
414 		    down_read_trylock(&adev->reset_sem)) {  in amdgpu_device_wreg()
416 			up_read(&adev->reset_sem);  in amdgpu_device_wreg()
418 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_device_wreg()
421 		adev->pcie_wreg(adev, reg * 4, v);  in amdgpu_device_wreg()
424 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);  in amdgpu_device_wreg()
428  * amdgpu_mm_wreg_mmio_rlc -  write register either with mmio or with RLC path if in range
435 	if (adev->in_pci_err_recovery)  in amdgpu_mm_wreg_mmio_rlc()
439 	    adev->gfx.rlc.funcs &&  in amdgpu_mm_wreg_mmio_rlc()
440 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {  in amdgpu_mm_wreg_mmio_rlc()
441 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))  in amdgpu_mm_wreg_mmio_rlc()
442 			return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);  in amdgpu_mm_wreg_mmio_rlc()
444 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  in amdgpu_mm_wreg_mmio_rlc()
449  * amdgpu_io_rreg - read an IO register
452  * @reg: dword aligned register offset
454  * Returns the 32 bit value from the offset specified.
458 	if (adev->in_pci_err_recovery)  in amdgpu_io_rreg()
461 	if ((reg * 4) < adev->rio_mem_size)  in amdgpu_io_rreg()
462 		return ioread32(adev->rio_mem + (reg * 4));  in amdgpu_io_rreg()
464 		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));  in amdgpu_io_rreg()
465 		return ioread32(adev->rio_mem + (mmMM_DATA * 4));  in amdgpu_io_rreg()
470  * amdgpu_io_wreg - write to an IO register
473  * @reg: dword aligned register offset
476  * Writes the value specified to the offset specified.
480 	if (adev->in_pci_err_recovery)  in amdgpu_io_wreg()
483 	if ((reg * 4) < adev->rio_mem_size)  in amdgpu_io_wreg()
484 		iowrite32(v, adev->rio_mem + (reg * 4));  in amdgpu_io_wreg()
486 		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));  in amdgpu_io_wreg()
487 		iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));  in amdgpu_io_wreg()
492  * amdgpu_mm_rdoorbell - read a doorbell dword
502 	if (adev->in_pci_err_recovery)  in amdgpu_mm_rdoorbell()
505 	if (index < adev->doorbell.num_doorbells) {  in amdgpu_mm_rdoorbell()
506 		return readl(adev->doorbell.ptr + index);  in amdgpu_mm_rdoorbell()
514  * amdgpu_mm_wdoorbell - write a doorbell dword
525 	if (adev->in_pci_err_recovery)  in amdgpu_mm_wdoorbell()
528 	if (index < adev->doorbell.num_doorbells) {  in amdgpu_mm_wdoorbell()
529 		writel(v, adev->doorbell.ptr + index);  in amdgpu_mm_wdoorbell()
536  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
546 	if (adev->in_pci_err_recovery)  in amdgpu_mm_rdoorbell64()
549 	if (index < adev->doorbell.num_doorbells) {  in amdgpu_mm_rdoorbell64()
550 		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));  in amdgpu_mm_rdoorbell64()
558  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
569 	if (adev->in_pci_err_recovery)  in amdgpu_mm_wdoorbell64()
572 	if (index < adev->doorbell.num_doorbells) {  in amdgpu_mm_wdoorbell64()
573 		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);  in amdgpu_mm_wdoorbell64()
580  * amdgpu_device_indirect_rreg - read an indirect register
583  * @pcie_index: mmio register offset
584  * @pcie_data: mmio register offset
597 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg()
598 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg()
599 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg()
604 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg()
610  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
613  * @pcie_index: mmio register offset
614  * @pcie_data: mmio register offset
627 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64()
628 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_rreg64()
629 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_rreg64()
639 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_rreg64()
645  * amdgpu_device_indirect_wreg - write an indirect register address
648  * @pcie_index: mmio register offset
649  * @pcie_data: mmio register offset
650  * @reg_addr: indirect register offset
662 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg()
663 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg()
664 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg()
670 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg()
674  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
677  * @pcie_index: mmio register offset
678  * @pcie_data: mmio register offset
679  * @reg_addr: indirect register offset
691 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64()
692 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;  in amdgpu_device_indirect_wreg64()
693 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;  in amdgpu_device_indirect_wreg64()
705 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);  in amdgpu_device_indirect_wreg64()
709  * amdgpu_invalid_rreg - dummy reg read function
712  * @reg: offset of register
726  * amdgpu_invalid_wreg - dummy reg write function
729  * @reg: offset of register
743  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
746  * @reg: offset of register
760  * amdgpu_invalid_wreg64 - dummy reg write function
763  * @reg: offset of register
777  * amdgpu_block_invalid_rreg - dummy reg read function
780  * @block: offset of instance
781  * @reg: offset of register
797  * amdgpu_block_invalid_wreg - dummy reg write function
800  * @block: offset of instance
801  * @reg: offset of register
817  * amdgpu_device_asic_init - Wrapper for atom asic_init
827 	return amdgpu_atom_asic_init(adev->mode_info.atom_context);  in amdgpu_device_asic_init()
831  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
842 				       &adev->vram_scratch.robj,  in amdgpu_device_vram_scratch_init()
843 				       &adev->vram_scratch.gpu_addr,  in amdgpu_device_vram_scratch_init()
844 				       (void **)&adev->vram_scratch.ptr);  in amdgpu_device_vram_scratch_init()
848  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
856 	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);  in amdgpu_device_vram_scratch_fini()
860  * amdgpu_device_program_register_sequence - program an array of registers.
889 			if (adev->family >= AMDGPU_FAMILY_AI)  in amdgpu_device_program_register_sequence()
899  * amdgpu_device_pci_config_reset - reset the GPU
908 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);  in amdgpu_device_pci_config_reset()
915  * amdgpu_device_doorbell_init - Init doorbell driver information.
926 	if (adev->asic_type < CHIP_BONAIRE) {  in amdgpu_device_doorbell_init()
927 		adev->doorbell.base = 0;  in amdgpu_device_doorbell_init()
928 		adev->doorbell.size = 0;  in amdgpu_device_doorbell_init()
929 		adev->doorbell.num_doorbells = 0;  in amdgpu_device_doorbell_init()
930 		adev->doorbell.ptr = NULL;  in amdgpu_device_doorbell_init()
934 	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)  in amdgpu_device_doorbell_init()
935 		return -EINVAL;  in amdgpu_device_doorbell_init()
940 	adev->doorbell.base = pci_resource_start(adev->pdev, 2);  in amdgpu_device_doorbell_init()
941 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);  in amdgpu_device_doorbell_init()
943 	adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),  in amdgpu_device_doorbell_init()
944 					     adev->doorbell_index.max_assignment+1);  in amdgpu_device_doorbell_init()
945 	if (adev->doorbell.num_doorbells == 0)  in amdgpu_device_doorbell_init()
946 		return -EINVAL;  in amdgpu_device_doorbell_init()
954 	if (adev->asic_type >= CHIP_VEGA10)  in amdgpu_device_doorbell_init()
955 		adev->doorbell.num_doorbells += 0x400;  in amdgpu_device_doorbell_init()
957 	adev->doorbell.ptr = ioremap(adev->doorbell.base,  in amdgpu_device_doorbell_init()
958 				     adev->doorbell.num_doorbells *  in amdgpu_device_doorbell_init()
960 	if (adev->doorbell.ptr == NULL)  in amdgpu_device_doorbell_init()
961 		return -ENOMEM;  in amdgpu_device_doorbell_init()
967  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
975 	iounmap(adev->doorbell.ptr);  in amdgpu_device_doorbell_fini()
976 	adev->doorbell.ptr = NULL;  in amdgpu_device_doorbell_fini()
988  * amdgpu_device_wb_fini - Disable Writeback and free memory
997 	if (adev->wb.wb_obj) {  in amdgpu_device_wb_fini()
998 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,  in amdgpu_device_wb_fini()
999 				      &adev->wb.gpu_addr,  in amdgpu_device_wb_fini()
1000 				      (void **)&adev->wb.wb);  in amdgpu_device_wb_fini()
1001 		adev->wb.wb_obj = NULL;  in amdgpu_device_wb_fini()
1006  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1012  * Returns 0 on success or an -error on failure.
1018 	if (adev->wb.wb_obj == NULL) {  in amdgpu_device_wb_init()
1022 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,  in amdgpu_device_wb_init()
1023 					    (void **)&adev->wb.wb);  in amdgpu_device_wb_init()
1025 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);  in amdgpu_device_wb_init()
1029 		adev->wb.num_wb = AMDGPU_MAX_WB;  in amdgpu_device_wb_init()
1030 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));  in amdgpu_device_wb_init()
1033 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);  in amdgpu_device_wb_init()
1040  * amdgpu_device_wb_get - Allocate a wb entry
1046  * Returns 0 on success or -EINVAL on failure.
1050 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);  in amdgpu_device_wb_get()  local
1052 	if (offset < adev->wb.num_wb) {  in amdgpu_device_wb_get()
1053 		__set_bit(offset, adev->wb.used);  in amdgpu_device_wb_get()
1054 		*wb = offset << 3; /* convert to dw offset */  in amdgpu_device_wb_get()
1057 		return -EINVAL;  in amdgpu_device_wb_get()
1062  * amdgpu_device_wb_free - Free a wb entry
1072 	if (wb < adev->wb.num_wb)  in amdgpu_device_wb_free()
1073 		__clear_bit(wb, adev->wb.used);  in amdgpu_device_wb_free()
1077  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1083  * driver loading by returning -ENODEV.
1087 	u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);  in amdgpu_device_resize_fb_bar()
1088 	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;  in amdgpu_device_resize_fb_bar()
1100 	if (adev->gmc.real_vram_size &&  in amdgpu_device_resize_fb_bar()
1101 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))  in amdgpu_device_resize_fb_bar()
1105 	root = adev->pdev->bus;  in amdgpu_device_resize_fb_bar()
1106 	while (root->parent)  in amdgpu_device_resize_fb_bar()
1107 		root = root->parent;  in amdgpu_device_resize_fb_bar()
1110 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&  in amdgpu_device_resize_fb_bar()
1111 		    res->start > 0x100000000ull)  in amdgpu_device_resize_fb_bar()
1120 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);  in amdgpu_device_resize_fb_bar()
1121 	pci_write_config_word(adev->pdev, PCI_COMMAND,  in amdgpu_device_resize_fb_bar()
1126 	if (adev->asic_type >= CHIP_BONAIRE)  in amdgpu_device_resize_fb_bar()
1127 		pci_release_resource(adev->pdev, 2);  in amdgpu_device_resize_fb_bar()
1129 	pci_release_resource(adev->pdev, 0);  in amdgpu_device_resize_fb_bar()
1131 	r = pci_resize_resource(adev->pdev, 0, rbar_size);  in amdgpu_device_resize_fb_bar()
1132 	if (r == -ENOSPC)  in amdgpu_device_resize_fb_bar()
1134 	else if (r && r != -ENOTSUPP)  in amdgpu_device_resize_fb_bar()
1137 	pci_assign_unassigned_bus_resources(adev->pdev->bus);  in amdgpu_device_resize_fb_bar()
1143 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))  in amdgpu_device_resize_fb_bar()
1144 		return -ENODEV;  in amdgpu_device_resize_fb_bar()
1146 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);  in amdgpu_device_resize_fb_bar()
1155  * amdgpu_device_need_post - check if the hw need post or not
1171 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot  in amdgpu_device_need_post()
1176 		if (adev->asic_type == CHIP_FIJI) {  in amdgpu_device_need_post()
1179 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);  in amdgpu_device_need_post()
1184 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);  in amdgpu_device_need_post()
1190 	if (adev->has_hw_reset) {  in amdgpu_device_need_post()
1191 		adev->has_hw_reset = false;  in amdgpu_device_need_post()
1196 	if (adev->asic_type >= CHIP_BONAIRE)  in amdgpu_device_need_post()
1210  * amdgpu_device_vga_set_decode - enable/disable vga decode
1230  * amdgpu_device_check_block_size - validate the vm block size
1236  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1242 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the  in amdgpu_device_check_block_size()
1244 	if (amdgpu_vm_block_size == -1)  in amdgpu_device_check_block_size()
1248 		dev_warn(adev->dev, "VM page table size (%d) too small\n",  in amdgpu_device_check_block_size()
1250 		amdgpu_vm_block_size = -1;  in amdgpu_device_check_block_size()
1255  * amdgpu_device_check_vm_size - validate the vm size
1265 	if (amdgpu_vm_size == -1)  in amdgpu_device_check_vm_size()
1269 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",  in amdgpu_device_check_vm_size()
1271 		amdgpu_vm_size = -1;  in amdgpu_device_check_vm_size()
1287 		DRM_WARN("Not 64-bit OS, feature not supported\n");  in amdgpu_device_check_smu_prv_buffer_size()
1305 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;  in amdgpu_device_check_smu_prv_buffer_size()
1312 	adev->pm.smu_prv_buffer_size = 0;  in amdgpu_device_check_smu_prv_buffer_size()
1316  * amdgpu_device_check_arguments - validate module params
1326 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",  in amdgpu_device_check_arguments()
1330 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",  in amdgpu_device_check_arguments()
1335 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {  in amdgpu_device_check_arguments()
1337 		dev_warn(adev->dev, "gart size (%d) too small\n",  in amdgpu_device_check_arguments()
1339 		amdgpu_gart_size = -1;  in amdgpu_device_check_arguments()
1342 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {  in amdgpu_device_check_arguments()
1344 		dev_warn(adev->dev, "gtt size (%d) too small\n",  in amdgpu_device_check_arguments()
1346 		amdgpu_gtt_size = -1;  in amdgpu_device_check_arguments()
1350 	if (amdgpu_vm_fragment_size != -1 &&  in amdgpu_device_check_arguments()
1352 		dev_warn(adev->dev, "valid range is between 4 and 9\n");  in amdgpu_device_check_arguments()
1353 		amdgpu_vm_fragment_size = -1;  in amdgpu_device_check_arguments()
1357 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",  in amdgpu_device_check_arguments()
1361 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",  in amdgpu_device_check_arguments()
1372 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);  in amdgpu_device_check_arguments()
1376 	if (amdgpu_num_kcq == -1) {  in amdgpu_device_check_arguments()
1380 …dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by use…  in amdgpu_device_check_arguments()
1389  * amdgpu_switcheroo_set_state - set switcheroo state
1408 		/* don't suspend or resume card normally */  in amdgpu_switcheroo_set_state()
1409 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;  in amdgpu_switcheroo_set_state()
1411 		pci_set_power_state(dev->pdev, PCI_D0);  in amdgpu_switcheroo_set_state()
1412 		amdgpu_device_load_pci_state(dev->pdev);  in amdgpu_switcheroo_set_state()
1413 		r = pci_enable_device(dev->pdev);  in amdgpu_switcheroo_set_state()
1418 		dev->switch_power_state = DRM_SWITCH_POWER_ON;  in amdgpu_switcheroo_set_state()
1423 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;  in amdgpu_switcheroo_set_state()
1425 		amdgpu_device_cache_pci_state(dev->pdev);  in amdgpu_switcheroo_set_state()
1427 		pci_disable_device(dev->pdev);  in amdgpu_switcheroo_set_state()
1428 		pci_set_power_state(dev->pdev, PCI_D3cold);  in amdgpu_switcheroo_set_state()
1429 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;  in amdgpu_switcheroo_set_state()
1434  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1451 	return atomic_read(&dev->open_count) == 0;  in amdgpu_switcheroo_can_switch()
1461  * amdgpu_device_ip_set_clockgating_state - set the CG state
1478 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_set_clockgating_state()
1479 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_set_clockgating_state()
1481 		if (adev->ip_blocks[i].version->type != block_type)  in amdgpu_device_ip_set_clockgating_state()
1483 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)  in amdgpu_device_ip_set_clockgating_state()
1485 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(  in amdgpu_device_ip_set_clockgating_state()
1489 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_set_clockgating_state()
1495  * amdgpu_device_ip_set_powergating_state - set the PG state
1512 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_set_powergating_state()
1513 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_set_powergating_state()
1515 		if (adev->ip_blocks[i].version->type != block_type)  in amdgpu_device_ip_set_powergating_state()
1517 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)  in amdgpu_device_ip_set_powergating_state()
1519 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(  in amdgpu_device_ip_set_powergating_state()
1523 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_set_powergating_state()
1529  * amdgpu_device_ip_get_clockgating_state - get the CG state
1544 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_get_clockgating_state()
1545 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_get_clockgating_state()
1547 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)  in amdgpu_device_ip_get_clockgating_state()
1548 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);  in amdgpu_device_ip_get_clockgating_state()
1553  * amdgpu_device_ip_wait_for_idle - wait for idle
1566 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_wait_for_idle()
1567 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_wait_for_idle()
1569 		if (adev->ip_blocks[i].version->type == block_type) {  in amdgpu_device_ip_wait_for_idle()
1570 			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);  in amdgpu_device_ip_wait_for_idle()
1581  * amdgpu_device_ip_is_idle - is the hardware IP idle
1594 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_is_idle()
1595 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_is_idle()
1597 		if (adev->ip_blocks[i].version->type == block_type)  in amdgpu_device_ip_is_idle()
1598 			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);  in amdgpu_device_ip_is_idle()
1605  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1619 	for (i = 0; i < adev->num_ip_blocks; i++)  in amdgpu_device_ip_get_ip_block()
1620 		if (adev->ip_blocks[i].version->type == type)  in amdgpu_device_ip_get_ip_block()
1621 			return &adev->ip_blocks[i];  in amdgpu_device_ip_get_ip_block()
1643 	if (ip_block && ((ip_block->version->major > major) ||  in amdgpu_device_ip_block_version_cmp()
1644 			((ip_block->version->major == major) &&  in amdgpu_device_ip_block_version_cmp()
1645 			(ip_block->version->minor >= minor))))  in amdgpu_device_ip_block_version_cmp()
1664 		return -EINVAL;  in amdgpu_device_ip_block_add()
1666 	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,  in amdgpu_device_ip_block_add()
1667 		  ip_block_version->funcs->name);  in amdgpu_device_ip_block_add()
1669 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;  in amdgpu_device_ip_block_add()
1675  * amdgpu_device_enable_virtual_display - enable virtual display feature
1688 	adev->enable_virtual_display = false;  in amdgpu_device_enable_virtual_display()
1692 		const char *pci_address_name = pci_name(ddev->pdev);  in amdgpu_device_enable_virtual_display()
1702 				int res = -1;  in amdgpu_device_enable_virtual_display()
1704 				adev->enable_virtual_display = true;  in amdgpu_device_enable_virtual_display()
1715 					adev->mode_info.num_crtc = num_crtc;  in amdgpu_device_enable_virtual_display()
1717 					adev->mode_info.num_crtc = 1;  in amdgpu_device_enable_virtual_display()
1725 			 adev->enable_virtual_display, adev->mode_info.num_crtc);  in amdgpu_device_enable_virtual_display()
1732  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1739  * Returns 0 on success, -EINVAL on failure.
1748 	adev->firmware.gpu_info_fw = NULL;  in amdgpu_device_parse_gpu_info_fw()
1750 	if (adev->mman.discovery_bin) {  in amdgpu_device_parse_gpu_info_fw()
1758 		if (adev->asic_type != CHIP_NAVI12)  in amdgpu_device_parse_gpu_info_fw()
1762 	switch (adev->asic_type) {  in amdgpu_device_parse_gpu_info_fw()
1798 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)  in amdgpu_device_parse_gpu_info_fw()
1800 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)  in amdgpu_device_parse_gpu_info_fw()
1809 		if (adev->apu_flags & AMD_APU_IS_RENOIR)  in amdgpu_device_parse_gpu_info_fw()
1826 	err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);  in amdgpu_device_parse_gpu_info_fw()
1828 		dev_err(adev->dev,  in amdgpu_device_parse_gpu_info_fw()
1833 	err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);  in amdgpu_device_parse_gpu_info_fw()
1835 		dev_err(adev->dev,  in amdgpu_device_parse_gpu_info_fw()
1841 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;  in amdgpu_device_parse_gpu_info_fw()
1842 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);  in amdgpu_device_parse_gpu_info_fw()
1844 	switch (hdr->version_major) {  in amdgpu_device_parse_gpu_info_fw()
1848 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
1849 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in amdgpu_device_parse_gpu_info_fw()
1854 		if (adev->asic_type == CHIP_NAVI12)  in amdgpu_device_parse_gpu_info_fw()
1857 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);  in amdgpu_device_parse_gpu_info_fw()
1858 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);  in amdgpu_device_parse_gpu_info_fw()
1859 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);  in amdgpu_device_parse_gpu_info_fw()
1860 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);  in amdgpu_device_parse_gpu_info_fw()
1861 		adev->gfx.config.max_texture_channel_caches =  in amdgpu_device_parse_gpu_info_fw()
1862 			le32_to_cpu(gpu_info_fw->gc_num_tccs);  in amdgpu_device_parse_gpu_info_fw()
1863 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);  in amdgpu_device_parse_gpu_info_fw()
1864 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);  in amdgpu_device_parse_gpu_info_fw()
1865 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);  in amdgpu_device_parse_gpu_info_fw()
1866 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);  in amdgpu_device_parse_gpu_info_fw()
1867 		adev->gfx.config.double_offchip_lds_buf =  in amdgpu_device_parse_gpu_info_fw()
1868 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);  in amdgpu_device_parse_gpu_info_fw()
1869 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);  in amdgpu_device_parse_gpu_info_fw()
1870 		adev->gfx.cu_info.max_waves_per_simd =  in amdgpu_device_parse_gpu_info_fw()
1871 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);  in amdgpu_device_parse_gpu_info_fw()
1872 		adev->gfx.cu_info.max_scratch_slots_per_cu =  in amdgpu_device_parse_gpu_info_fw()
1873 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);  in amdgpu_device_parse_gpu_info_fw()
1874 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);  in amdgpu_device_parse_gpu_info_fw()
1875 		if (hdr->version_minor >= 1) {  in amdgpu_device_parse_gpu_info_fw()
1877 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
1878 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in amdgpu_device_parse_gpu_info_fw()
1879 			adev->gfx.config.num_sc_per_sh =  in amdgpu_device_parse_gpu_info_fw()
1880 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);  in amdgpu_device_parse_gpu_info_fw()
1881 			adev->gfx.config.num_packer_per_sc =  in amdgpu_device_parse_gpu_info_fw()
1882 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);  in amdgpu_device_parse_gpu_info_fw()
1890 		if (hdr->version_minor == 2) {  in amdgpu_device_parse_gpu_info_fw()
1892 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +  in amdgpu_device_parse_gpu_info_fw()
1893 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in amdgpu_device_parse_gpu_info_fw()
1894 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;  in amdgpu_device_parse_gpu_info_fw()
1899 		dev_err(adev->dev,  in amdgpu_device_parse_gpu_info_fw()
1900 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);  in amdgpu_device_parse_gpu_info_fw()
1901 		err = -EINVAL;  in amdgpu_device_parse_gpu_info_fw()
1909  * amdgpu_device_ip_early_init - run early init for hardware IPs
1930 	switch (adev->asic_type) {  in amdgpu_device_ip_early_init()
1937 		adev->family = AMDGPU_FAMILY_SI;  in amdgpu_device_ip_early_init()
1949 		if (adev->flags & AMD_IS_APU)  in amdgpu_device_ip_early_init()
1950 			adev->family = AMDGPU_FAMILY_KV;  in amdgpu_device_ip_early_init()
1952 			adev->family = AMDGPU_FAMILY_CI;  in amdgpu_device_ip_early_init()
1968 		if (adev->flags & AMD_IS_APU)  in amdgpu_device_ip_early_init()
1969 			adev->family = AMDGPU_FAMILY_CZ;  in amdgpu_device_ip_early_init()
1971 			adev->family = AMDGPU_FAMILY_VI;  in amdgpu_device_ip_early_init()
1983 		if (adev->flags & AMD_IS_APU)  in amdgpu_device_ip_early_init()
1984 			adev->family = AMDGPU_FAMILY_RV;  in amdgpu_device_ip_early_init()
1986 			adev->family = AMDGPU_FAMILY_AI;  in amdgpu_device_ip_early_init()
1997 		adev->family = AMDGPU_FAMILY_NV;  in amdgpu_device_ip_early_init()
2005 		return -EINVAL;  in amdgpu_device_ip_early_init()
2010 	adev->pm.pp_feature = amdgpu_pp_feature_mask;  in amdgpu_device_ip_early_init()
2012 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;  in amdgpu_device_ip_early_init()
2014 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_early_init()
2017 				  i, adev->ip_blocks[i].version->funcs->name);  in amdgpu_device_ip_early_init()
2018 			adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_early_init()
2020 			if (adev->ip_blocks[i].version->funcs->early_init) {  in amdgpu_device_ip_early_init()
2021 				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);  in amdgpu_device_ip_early_init()
2022 				if (r == -ENOENT) {  in amdgpu_device_ip_early_init()
2023 					adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_early_init()
2026 						  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_early_init()
2029 					adev->ip_blocks[i].status.valid = true;  in amdgpu_device_ip_early_init()
2032 				adev->ip_blocks[i].status.valid = true;  in amdgpu_device_ip_early_init()
2036 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {  in amdgpu_device_ip_early_init()
2043 				return -EINVAL;  in amdgpu_device_ip_early_init()
2047 				dev_err(adev->dev, "amdgpu_atombios_init failed\n");  in amdgpu_device_ip_early_init()
2054 	adev->cg_flags &= amdgpu_cg_mask;  in amdgpu_device_ip_early_init()
2055 	adev->pg_flags &= amdgpu_pg_mask;  in amdgpu_device_ip_early_init()
2064 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_hw_init_phase1()
2065 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_hw_init_phase1()
2067 		if (adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_hw_init_phase1()
2069 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_hw_init_phase1()
2070 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||  in amdgpu_device_ip_hw_init_phase1()
2071 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {  in amdgpu_device_ip_hw_init_phase1()
2072 			r = adev->ip_blocks[i].version->funcs->hw_init(adev);  in amdgpu_device_ip_hw_init_phase1()
2075 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_hw_init_phase1()
2078 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_hw_init_phase1()
2089 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_hw_init_phase2()
2090 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_hw_init_phase2()
2092 		if (adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_hw_init_phase2()
2094 		r = adev->ip_blocks[i].version->funcs->hw_init(adev);  in amdgpu_device_ip_hw_init_phase2()
2097 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_hw_init_phase2()
2100 		adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_hw_init_phase2()
2112 	if (adev->asic_type >= CHIP_VEGA10) {  in amdgpu_device_fw_loading()
2113 		for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_fw_loading()
2114 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)  in amdgpu_device_fw_loading()
2118 			if (adev->ip_blocks[i].status.hw == true)  in amdgpu_device_fw_loading()
2121 			if (amdgpu_in_reset(adev) || adev->in_suspend) {  in amdgpu_device_fw_loading()
2122 				r = adev->ip_blocks[i].version->funcs->resume(adev);  in amdgpu_device_fw_loading()
2124 					DRM_ERROR("resume of IP block <%s> failed %d\n",  in amdgpu_device_fw_loading()
2125 							  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_fw_loading()
2129 				r = adev->ip_blocks[i].version->funcs->hw_init(adev);  in amdgpu_device_fw_loading()
2132 							  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_fw_loading()
2137 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_fw_loading()
2142 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)  in amdgpu_device_fw_loading()
2149  * amdgpu_device_ip_init - run init for hardware IPs
2167 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_init()
2168 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_init()
2170 		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);  in amdgpu_device_ip_init()
2173 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_init()
2176 		adev->ip_blocks[i].status.sw = true;  in amdgpu_device_ip_init()
2179 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {  in amdgpu_device_ip_init()
2185 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);  in amdgpu_device_ip_init()
2195 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_init()
2199 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,  in amdgpu_device_ip_init()
2215 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);  in amdgpu_device_ip_init()
2255 	if (adev->gmc.xgmi.num_physical_nodes > 1)  in amdgpu_device_ip_init()
2269  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2279 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);  in amdgpu_device_fill_reset_magic()
2283  * amdgpu_device_check_vram_lost - check if vram is valid
2294 	if (memcmp(adev->gart.ptr, adev->reset_magic,  in amdgpu_device_check_vram_lost()
2315  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2335 	for (j = 0; j < adev->num_ip_blocks; j++) {  in amdgpu_device_set_cg_state()
2336 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;  in amdgpu_device_set_cg_state()
2337 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_set_cg_state()
2340 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&  in amdgpu_device_set_cg_state()
2341 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&  in amdgpu_device_set_cg_state()
2342 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&  in amdgpu_device_set_cg_state()
2343 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&  in amdgpu_device_set_cg_state()
2344 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {  in amdgpu_device_set_cg_state()
2346 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,  in amdgpu_device_set_cg_state()
2350 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_set_cg_state()
2366 	for (j = 0; j < adev->num_ip_blocks; j++) {  in amdgpu_device_set_pg_state()
2367 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;  in amdgpu_device_set_pg_state()
2368 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_set_pg_state()
2371 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&  in amdgpu_device_set_pg_state()
2372 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&  in amdgpu_device_set_pg_state()
2373 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&  in amdgpu_device_set_pg_state()
2374 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&  in amdgpu_device_set_pg_state()
2375 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {  in amdgpu_device_set_pg_state()
2377 			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,  in amdgpu_device_set_pg_state()
2381 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_set_pg_state()
2407 		adev = gpu_ins->adev;  in amdgpu_device_enable_mgpu_fan_boost()
2408 		if (!(adev->flags & AMD_IS_APU) &&  in amdgpu_device_enable_mgpu_fan_boost()
2409 		    !gpu_ins->mgpu_fan_enabled) {  in amdgpu_device_enable_mgpu_fan_boost()
2414 			gpu_ins->mgpu_fan_enabled = 1;  in amdgpu_device_enable_mgpu_fan_boost()
2425  * amdgpu_device_ip_late_init - run late init for hardware IPs
2441 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_late_init()
2442 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_late_init()
2444 		if (adev->ip_blocks[i].version->funcs->late_init) {  in amdgpu_device_ip_late_init()
2445 			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);  in amdgpu_device_ip_late_init()
2448 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_late_init()
2452 		adev->ip_blocks[i].status.late_initialized = true;  in amdgpu_device_ip_late_init()
2467 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_device_ip_late_init()
2471 		 * Reset device p-state to low as this was booted with high.  in amdgpu_device_ip_late_init()
2483 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {  in amdgpu_device_ip_late_init()
2486 				if (gpu_instance->adev->flags & AMD_IS_APU)  in amdgpu_device_ip_late_init()
2489 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,  in amdgpu_device_ip_late_init()
2505  * amdgpu_device_ip_fini - run fini for hardware IPs
2519 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)  in amdgpu_device_ip_fini()
2524 	if (adev->gmc.xgmi.num_physical_nodes > 1)  in amdgpu_device_ip_fini()
2533 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_fini()
2534 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_fini()
2536 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {  in amdgpu_device_ip_fini()
2537 			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);  in amdgpu_device_ip_fini()
2541 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_fini()
2543 			adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_fini()
2548 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini()
2549 		if (!adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_fini()
2552 		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);  in amdgpu_device_ip_fini()
2556 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_fini()
2559 		adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_fini()
2563 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini()
2564 		if (!adev->ip_blocks[i].status.sw)  in amdgpu_device_ip_fini()
2567 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {  in amdgpu_device_ip_fini()
2569 			amdgpu_free_static_csa(&adev->virt.csa_obj);  in amdgpu_device_ip_fini()
2575 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);  in amdgpu_device_ip_fini()
2579 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_fini()
2581 		adev->ip_blocks[i].status.sw = false;  in amdgpu_device_ip_fini()
2582 		adev->ip_blocks[i].status.valid = false;  in amdgpu_device_ip_fini()
2585 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_fini()
2586 		if (!adev->ip_blocks[i].status.late_initialized)  in amdgpu_device_ip_fini()
2588 		if (adev->ip_blocks[i].version->funcs->late_fini)  in amdgpu_device_ip_fini()
2589 			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);  in amdgpu_device_ip_fini()
2590 		adev->ip_blocks[i].status.late_initialized = false;  in amdgpu_device_ip_fini()
2603  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2623 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_device_delay_enable_gfx_off()
2624 	if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {  in amdgpu_device_delay_enable_gfx_off()
2626 			adev->gfx.gfx_off_state = true;  in amdgpu_device_delay_enable_gfx_off()
2628 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_device_delay_enable_gfx_off()
2632  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2649 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_suspend_phase1()
2650 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_suspend_phase1()
2654 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)  in amdgpu_device_ip_suspend_phase1()
2658 		r = adev->ip_blocks[i].version->funcs->suspend(adev);  in amdgpu_device_ip_suspend_phase1()
2662 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_suspend_phase1()
2666 		adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase1()
2673  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2687 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {  in amdgpu_device_ip_suspend_phase2()
2688 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_suspend_phase2()
2691 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)  in amdgpu_device_ip_suspend_phase2()
2695 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {  in amdgpu_device_ip_suspend_phase2()
2696 			adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase2()
2700 		r = adev->ip_blocks[i].version->funcs->suspend(adev);  in amdgpu_device_ip_suspend_phase2()
2704 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_suspend_phase2()
2706 		adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase2()
2709 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {  in amdgpu_device_ip_suspend_phase2()
2710 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);  in amdgpu_device_ip_suspend_phase2()
2713 							adev->mp1_state, r);  in amdgpu_device_ip_suspend_phase2()
2718 		adev->ip_blocks[i].status.hw = false;  in amdgpu_device_ip_suspend_phase2()
2725  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2768 		block = &adev->ip_blocks[i];  in amdgpu_device_ip_reinit_early_sriov()
2769 		block->status.hw = false;  in amdgpu_device_ip_reinit_early_sriov()
2773 			if (block->version->type != ip_order[j] ||  in amdgpu_device_ip_reinit_early_sriov()
2774 				!block->status.valid)  in amdgpu_device_ip_reinit_early_sriov()
2777 			r = block->version->funcs->hw_init(adev);  in amdgpu_device_ip_reinit_early_sriov()
2778 			DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");  in amdgpu_device_ip_reinit_early_sriov()
2781 			block->status.hw = true;  in amdgpu_device_ip_reinit_early_sriov()
2806 		for (j = 0; j < adev->num_ip_blocks; j++) {  in amdgpu_device_ip_reinit_late_sriov()
2807 			block = &adev->ip_blocks[j];  in amdgpu_device_ip_reinit_late_sriov()
2809 			if (block->version->type != ip_order[i] ||  in amdgpu_device_ip_reinit_late_sriov()
2810 				!block->status.valid ||  in amdgpu_device_ip_reinit_late_sriov()
2811 				block->status.hw)  in amdgpu_device_ip_reinit_late_sriov()
2814 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)  in amdgpu_device_ip_reinit_late_sriov()
2815 				r = block->version->funcs->resume(adev);  in amdgpu_device_ip_reinit_late_sriov()
2817 				r = block->version->funcs->hw_init(adev);  in amdgpu_device_ip_reinit_late_sriov()
2819 			DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");  in amdgpu_device_ip_reinit_late_sriov()
2822 			block->status.hw = true;  in amdgpu_device_ip_reinit_late_sriov()
2830  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2834  * First resume function for hardware IPs.  The list of all the hardware
2835  * IPs that make up the asic is walked and the resume callbacks are run for
2836  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
2845 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_resume_phase1()
2846 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_resume_phase1()
2848 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_resume_phase1()
2849 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||  in amdgpu_device_ip_resume_phase1()
2850 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {  in amdgpu_device_ip_resume_phase1()
2852 			r = adev->ip_blocks[i].version->funcs->resume(adev);  in amdgpu_device_ip_resume_phase1()
2854 				DRM_ERROR("resume of IP block <%s> failed %d\n",  in amdgpu_device_ip_resume_phase1()
2855 					  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_resume_phase1()
2858 			adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_resume_phase1()
2866  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2870  * First resume function for hardware IPs.  The list of all the hardware
2871  * IPs that make up the asic is walked and the resume callbacks are run for
2872  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
2882 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_resume_phase2()
2883 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)  in amdgpu_device_ip_resume_phase2()
2885 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||  in amdgpu_device_ip_resume_phase2()
2886 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||  in amdgpu_device_ip_resume_phase2()
2887 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||  in amdgpu_device_ip_resume_phase2()
2888 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)  in amdgpu_device_ip_resume_phase2()
2890 		r = adev->ip_blocks[i].version->funcs->resume(adev);  in amdgpu_device_ip_resume_phase2()
2892 			DRM_ERROR("resume of IP block <%s> failed %d\n",  in amdgpu_device_ip_resume_phase2()
2893 				  adev->ip_blocks[i].version->funcs->name, r);  in amdgpu_device_ip_resume_phase2()
2896 		adev->ip_blocks[i].status.hw = true;  in amdgpu_device_ip_resume_phase2()
2903  * amdgpu_device_ip_resume - run resume for hardware IPs
2907  * Main resume function for hardware IPs.  The hardware IPs
2908  * are split into two resume functions because they are
2932  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2936  * Query the VBIOS data tables to determine if the board supports SR-IOV.
2941 		if (adev->is_atom_fw) {  in amdgpu_device_detect_sriov_bios()
2943 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;  in amdgpu_device_detect_sriov_bios()
2946 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;  in amdgpu_device_detect_sriov_bios()
2949 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))  in amdgpu_device_detect_sriov_bios()
2955  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2980 		 * Fallback to the non-DC driver here by default so as not to  in amdgpu_device_asic_has_dc_support()
3018  * amdgpu_device_has_dc_support - check if dc is supported
3026 	if (amdgpu_sriov_vf(adev) || adev->enable_virtual_display)  in amdgpu_device_has_dc_support()
3029 	return amdgpu_device_asic_has_dc_support(adev->asic_type);  in amdgpu_device_has_dc_support()
3051 		task_barrier_enter(&hive->tb);  in amdgpu_device_xgmi_reset_func()
3052 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));  in amdgpu_device_xgmi_reset_func()
3054 		if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
3057 		task_barrier_exit(&hive->tb);  in amdgpu_device_xgmi_reset_func()
3058 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));  in amdgpu_device_xgmi_reset_func()
3060 		if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
3063 		if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)  in amdgpu_device_xgmi_reset_func()
3064 			adev->mmhub.funcs->reset_ras_error_count(adev);  in amdgpu_device_xgmi_reset_func()
3067 		task_barrier_full(&hive->tb);  in amdgpu_device_xgmi_reset_func()
3068 		adev->asic_reset_res =  amdgpu_asic_reset(adev);  in amdgpu_device_xgmi_reset_func()
3072 	if (adev->asic_reset_res)  in amdgpu_device_xgmi_reset_func()
3074 			 adev->asic_reset_res, adev_to_drm(adev)->unique);  in amdgpu_device_xgmi_reset_func()
3089 	 * In SR-IOV or passthrough mode, timeout for compute  in amdgpu_device_get_job_timeout_settings()
3092 	adev->gfx_timeout = msecs_to_jiffies(10000);  in amdgpu_device_get_job_timeout_settings()
3093 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
3095 		adev->compute_timeout =  msecs_to_jiffies(60000);  in amdgpu_device_get_job_timeout_settings()
3097 		adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;  in amdgpu_device_get_job_timeout_settings()
3117 				adev->gfx_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
3120 				adev->compute_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
3123 				adev->sdma_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
3126 				adev->video_timeout = timeout;  in amdgpu_device_get_job_timeout_settings()
3134 		 * it should apply to all non-compute jobs.  in amdgpu_device_get_job_timeout_settings()
3137 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
3139 				adev->compute_timeout = adev->gfx_timeout;  in amdgpu_device_get_job_timeout_settings()
3156  * amdgpu_device_init - initialize the driver
3169 	struct pci_dev *pdev = adev->pdev;  in amdgpu_device_init()
3174 	adev->shutdown = false;  in amdgpu_device_init()
3175 	adev->flags = flags;  in amdgpu_device_init()
3178 		adev->asic_type = amdgpu_force_asic_type;  in amdgpu_device_init()
3180 		adev->asic_type = flags & AMD_ASIC_MASK;  in amdgpu_device_init()
3182 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;  in amdgpu_device_init()
3184 		adev->usec_timeout *= 10;  in amdgpu_device_init()
3185 	adev->gmc.gart_size = 512 * 1024 * 1024;  in amdgpu_device_init()
3186 	adev->accel_working = false;  in amdgpu_device_init()
3187 	adev->num_rings = 0;  in amdgpu_device_init()
3188 	adev->mman.buffer_funcs = NULL;  in amdgpu_device_init()
3189 	adev->mman.buffer_funcs_ring = NULL;  in amdgpu_device_init()
3190 	adev->vm_manager.vm_pte_funcs = NULL;  in amdgpu_device_init()
3191 	adev->vm_manager.vm_pte_num_scheds = 0;  in amdgpu_device_init()
3192 	adev->gmc.gmc_funcs = NULL;  in amdgpu_device_init()
3193 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);  in amdgpu_device_init()
3194 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);  in amdgpu_device_init()
3196 	adev->smc_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3197 	adev->smc_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3198 	adev->pcie_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3199 	adev->pcie_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3200 	adev->pciep_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3201 	adev->pciep_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3202 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;  in amdgpu_device_init()
3203 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;  in amdgpu_device_init()
3204 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3205 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3206 	adev->didt_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3207 	adev->didt_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3208 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;  in amdgpu_device_init()
3209 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;  in amdgpu_device_init()
3210 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;  in amdgpu_device_init()
3211 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;  in amdgpu_device_init()
3214 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,  in amdgpu_device_init()
3215 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);  in amdgpu_device_init()
3219 	atomic_set(&adev->irq.ih.lock, 0);  in amdgpu_device_init()
3220 	mutex_init(&adev->firmware.mutex);  in amdgpu_device_init()
3221 	mutex_init(&adev->pm.mutex);  in amdgpu_device_init()
3222 	mutex_init(&adev->gfx.gpu_clock_mutex);  in amdgpu_device_init()
3223 	mutex_init(&adev->srbm_mutex);  in amdgpu_device_init()
3224 	mutex_init(&adev->gfx.pipe_reserve_mutex);  in amdgpu_device_init()
3225 	mutex_init(&adev->gfx.gfx_off_mutex);  in amdgpu_device_init()
3226 	mutex_init(&adev->grbm_idx_mutex);  in amdgpu_device_init()
3227 	mutex_init(&adev->mn_lock);  in amdgpu_device_init()
3228 	mutex_init(&adev->virt.vf_errors.lock);  in amdgpu_device_init()
3229 	hash_init(adev->mn_hash);  in amdgpu_device_init()
3230 	atomic_set(&adev->in_gpu_reset, 0);  in amdgpu_device_init()
3231 	init_rwsem(&adev->reset_sem);  in amdgpu_device_init()
3232 	mutex_init(&adev->psp.mutex);  in amdgpu_device_init()
3233 	mutex_init(&adev->notifier_lock);  in amdgpu_device_init()
3239 	spin_lock_init(&adev->mmio_idx_lock);  in amdgpu_device_init()
3240 	spin_lock_init(&adev->smc_idx_lock);  in amdgpu_device_init()
3241 	spin_lock_init(&adev->pcie_idx_lock);  in amdgpu_device_init()
3242 	spin_lock_init(&adev->uvd_ctx_idx_lock);  in amdgpu_device_init()
3243 	spin_lock_init(&adev->didt_idx_lock);  in amdgpu_device_init()
3244 	spin_lock_init(&adev->gc_cac_idx_lock);  in amdgpu_device_init()
3245 	spin_lock_init(&adev->se_cac_idx_lock);  in amdgpu_device_init()
3246 	spin_lock_init(&adev->audio_endpt_idx_lock);  in amdgpu_device_init()
3247 	spin_lock_init(&adev->mm_stats.lock);  in amdgpu_device_init()
3249 	INIT_LIST_HEAD(&adev->shadow_list);  in amdgpu_device_init()
3250 	mutex_init(&adev->shadow_list_lock);  in amdgpu_device_init()
3252 	INIT_DELAYED_WORK(&adev->delayed_init_work,  in amdgpu_device_init()
3254 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,  in amdgpu_device_init()
3257 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);  in amdgpu_device_init()
3259 	adev->gfx.gfx_off_req_count = 1;  in amdgpu_device_init()
3260 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;  in amdgpu_device_init()
3262 	atomic_set(&adev->throttling_logging_enabled, 1);  in amdgpu_device_init()
3265 	 * to avoid log flooding. "-1" is subtracted since the thermal  in amdgpu_device_init()
3270 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);  in amdgpu_device_init()
3271 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);  in amdgpu_device_init()
3275 	if (adev->asic_type >= CHIP_BONAIRE) {  in amdgpu_device_init()
3276 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);  in amdgpu_device_init()
3277 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);  in amdgpu_device_init()
3279 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);  in amdgpu_device_init()
3280 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);  in amdgpu_device_init()
3283 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);  in amdgpu_device_init()
3284 	if (adev->rmmio == NULL) {  in amdgpu_device_init()
3285 		return -ENOMEM;  in amdgpu_device_init()
3287 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);  in amdgpu_device_init()
3288 	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);  in amdgpu_device_init()
3292 		if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {  in amdgpu_device_init()
3293 			adev->rio_mem_size = pci_resource_len(adev->pdev, i);  in amdgpu_device_init()
3294 			adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);  in amdgpu_device_init()
3298 	if (adev->rio_mem == NULL)  in amdgpu_device_init()
3302 	r = pci_enable_atomic_ops_to_root(adev->pdev,  in amdgpu_device_init()
3306 		adev->have_atomics_support = false;  in amdgpu_device_init()
3309 		adev->have_atomics_support = true;  in amdgpu_device_init()
3317 	if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)  in amdgpu_device_init()
3318 		adev->enable_mes = true;  in amdgpu_device_init()
3325 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");  in amdgpu_device_init()
3340 	vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);  in amdgpu_device_init()
3347 	    !pci_is_thunderbolt_attached(adev->pdev))  in amdgpu_device_init()
3348 		vga_switcheroo_register_client(adev->pdev,  in amdgpu_device_init()
3351 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);  in amdgpu_device_init()
3368 			dev_err(adev->dev, "asic reset on init failed\n");  in amdgpu_device_init()
3373 	pci_enable_pcie_error_reporting(adev->ddev.pdev);  in amdgpu_device_init()
3377 		if (!adev->bios) {  in amdgpu_device_init()
3378 			dev_err(adev->dev, "no vBIOS found\n");  in amdgpu_device_init()
3379 			r = -EINVAL;  in amdgpu_device_init()
3385 			dev_err(adev->dev, "gpu post error!\n");  in amdgpu_device_init()
3390 	if (adev->is_atom_fw) {  in amdgpu_device_init()
3394 			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");  in amdgpu_device_init()
3402 			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");  in amdgpu_device_init()
3415 		dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");  in amdgpu_device_init()
3430 			dev_err(adev->dev, "VF exclusive mode timeout\n");  in amdgpu_device_init()
3432 			adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;  in amdgpu_device_init()
3433 			adev->virt.ops = NULL;  in amdgpu_device_init()
3434 			r = -EAGAIN;  in amdgpu_device_init()
3437 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");  in amdgpu_device_init()
3442 	dev_info(adev->dev,  in amdgpu_device_init()
3444 			adev->gfx.config.max_shader_engines,  in amdgpu_device_init()
3445 			adev->gfx.config.max_sh_per_se,  in amdgpu_device_init()
3446 			adev->gfx.config.max_cu_per_sh,  in amdgpu_device_init()
3447 			adev->gfx.cu_info.number);  in amdgpu_device_init()
3449 	adev->accel_working = true;  in amdgpu_device_init()
3459 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));  in amdgpu_device_init()
3465 		adev->pm_sysfs_en = false;  in amdgpu_device_init()
3468 		adev->pm_sysfs_en = true;  in amdgpu_device_init()
3472 		adev->ucode_sysfs_en = false;  in amdgpu_device_init()
3475 		adev->ucode_sysfs_en = true;  in amdgpu_device_init()
3478 		if (adev->accel_working)  in amdgpu_device_init()
3484 		if (adev->accel_working)  in amdgpu_device_init()
3502 		dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");  in amdgpu_device_init()
3510 	queue_delayed_work(system_wq, &adev->delayed_init_work,  in amdgpu_device_init()
3514 		flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_init()
3516 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);  in amdgpu_device_init()
3518 		dev_err(adev->dev, "Could not create amdgpu device attr\n");  in amdgpu_device_init()
3523 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");  in amdgpu_device_init()
3526 	if (amdgpu_device_cache_pci_state(adev->pdev))  in amdgpu_device_init()
3534 		vga_switcheroo_fini_domain_pm_ops(adev->dev);  in amdgpu_device_init()
3537 	iounmap(adev->rmmio);  in amdgpu_device_init()
3538 	adev->rmmio = NULL;  in amdgpu_device_init()
3544  * amdgpu_device_fini - tear down the driver
3553 	dev_info(adev->dev, "amdgpu: finishing device.\n");  in amdgpu_device_fini()
3554 	flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_fini()
3555 	adev->shutdown = true;  in amdgpu_device_fini()
3557 	kfree(adev->pci_state);  in amdgpu_device_fini()
3569 	if (adev->mode_info.mode_config_initialized){  in amdgpu_device_fini()
3576 	if (adev->pm_sysfs_en)  in amdgpu_device_fini()
3580 	release_firmware(adev->firmware.gpu_info_fw);  in amdgpu_device_fini()
3581 	adev->firmware.gpu_info_fw = NULL;  in amdgpu_device_fini()
3582 	adev->accel_working = false;  in amdgpu_device_fini()
3590 	kfree(adev->bios);  in amdgpu_device_fini()
3591 	adev->bios = NULL;  in amdgpu_device_fini()
3595 	    !pci_is_thunderbolt_attached(adev->pdev))  in amdgpu_device_fini()
3596 		vga_switcheroo_unregister_client(adev->pdev);  in amdgpu_device_fini()
3598 		vga_switcheroo_fini_domain_pm_ops(adev->dev);  in amdgpu_device_fini()
3599 	vga_client_register(adev->pdev, NULL, NULL, NULL);  in amdgpu_device_fini()
3600 	if (adev->rio_mem)  in amdgpu_device_fini()
3601 		pci_iounmap(adev->pdev, adev->rio_mem);  in amdgpu_device_fini()
3602 	adev->rio_mem = NULL;  in amdgpu_device_fini()
3603 	iounmap(adev->rmmio);  in amdgpu_device_fini()
3604 	adev->rmmio = NULL;  in amdgpu_device_fini()
3607 	if (adev->ucode_sysfs_en)  in amdgpu_device_fini()
3610 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);  in amdgpu_device_fini()
3613 	if (adev->mman.discovery_bin)  in amdgpu_device_fini()
3619  * Suspend & resume.
3622  * amdgpu_device_suspend - initiate device suspend
3641 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)  in amdgpu_device_suspend()
3644 	adev->in_suspend = true;  in amdgpu_device_suspend()
3650 	cancel_delayed_work_sync(&adev->delayed_init_work);  in amdgpu_device_suspend()
3662 		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {  in amdgpu_device_suspend()
3664 			struct drm_framebuffer *fb = crtc->primary->fb;  in amdgpu_device_suspend()
3667 			if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {  in amdgpu_device_suspend()
3668 				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);  in amdgpu_device_suspend()
3676 			if (fb == NULL || fb->obj[0] == NULL) {  in amdgpu_device_suspend()
3679 			robj = gem_to_amdgpu_bo(fb->obj[0]);  in amdgpu_device_suspend()
3714  * amdgpu_device_resume - initiate device resume
3717  * @fbcon : notify the fbdev of resume
3721  * Called at driver resume.
3731 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)  in amdgpu_device_resume()
3738 			dev_err(adev->dev, "amdgpu asic init failed\n");  in amdgpu_device_resume()
3743 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);  in amdgpu_device_resume()
3753 	queue_delayed_work(system_wq, &adev->delayed_init_work,  in amdgpu_device_resume()
3758 		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {  in amdgpu_device_resume()
3761 			if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {  in amdgpu_device_resume()
3762 				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);  in amdgpu_device_resume()
3767 						dev_err(adev->dev, "Failed to pin cursor BO (%d)\n", r);  in amdgpu_device_resume()
3768 					amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);  in amdgpu_device_resume()
3779 	flush_delayed_work(&adev->delayed_init_work);  in amdgpu_device_resume()
3815 	dev->dev->power.disable_depth++;  in amdgpu_device_resume()
3822 	dev->dev->power.disable_depth--;  in amdgpu_device_resume()
3824 	adev->in_suspend = false;  in amdgpu_device_resume()
3830  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3850 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_check_soft_reset()
3851 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_check_soft_reset()
3853 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)  in amdgpu_device_ip_check_soft_reset()
3854 			adev->ip_blocks[i].status.hang =  in amdgpu_device_ip_check_soft_reset()
3855 				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);  in amdgpu_device_ip_check_soft_reset()
3856 		if (adev->ip_blocks[i].status.hang) {  in amdgpu_device_ip_check_soft_reset()
3857 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);  in amdgpu_device_ip_check_soft_reset()
3865  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3879 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_pre_soft_reset()
3880 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_pre_soft_reset()
3882 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_pre_soft_reset()
3883 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {  in amdgpu_device_ip_pre_soft_reset()
3884 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);  in amdgpu_device_ip_pre_soft_reset()
3894  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3909 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_need_full_reset()
3910 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_need_full_reset()
3912 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||  in amdgpu_device_ip_need_full_reset()
3913 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||  in amdgpu_device_ip_need_full_reset()
3914 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||  in amdgpu_device_ip_need_full_reset()
3915 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||  in amdgpu_device_ip_need_full_reset()
3916 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {  in amdgpu_device_ip_need_full_reset()
3917 			if (adev->ip_blocks[i].status.hang) {  in amdgpu_device_ip_need_full_reset()
3918 				dev_info(adev->dev, "Some block need full reset!\n");  in amdgpu_device_ip_need_full_reset()
3927  * amdgpu_device_ip_soft_reset - do a soft reset
3941 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_soft_reset()
3942 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_soft_reset()
3944 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_soft_reset()
3945 		    adev->ip_blocks[i].version->funcs->soft_reset) {  in amdgpu_device_ip_soft_reset()
3946 			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);  in amdgpu_device_ip_soft_reset()
3956  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3970 	for (i = 0; i < adev->num_ip_blocks; i++) {  in amdgpu_device_ip_post_soft_reset()
3971 		if (!adev->ip_blocks[i].status.valid)  in amdgpu_device_ip_post_soft_reset()
3973 		if (adev->ip_blocks[i].status.hang &&  in amdgpu_device_ip_post_soft_reset()
3974 		    adev->ip_blocks[i].version->funcs->post_soft_reset)  in amdgpu_device_ip_post_soft_reset()
3975 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);  in amdgpu_device_ip_post_soft_reset()
3984  * amdgpu_device_recover_vram - Recover some VRAM contents
4006 	dev_info(adev->dev, "recover vram bo from shadow start\n");  in amdgpu_device_recover_vram()
4007 	mutex_lock(&adev->shadow_list_lock);  in amdgpu_device_recover_vram()
4008 	list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {  in amdgpu_device_recover_vram()
4011 		if (shadow->tbo.mem.mem_type != TTM_PL_TT ||  in amdgpu_device_recover_vram()
4012 		    shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||  in amdgpu_device_recover_vram()
4013 		    shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)  in amdgpu_device_recover_vram()
4025 				r = -ETIMEDOUT;  in amdgpu_device_recover_vram()
4035 	mutex_unlock(&adev->shadow_list_lock);  in amdgpu_device_recover_vram()
4042 		dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);  in amdgpu_device_recover_vram()
4043 		return -EIO;  in amdgpu_device_recover_vram()
4046 	dev_info(adev->dev, "recover vram bo from shadow done\n");  in amdgpu_device_recover_vram()
4052  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4074 	/* Resume IP prior to SMC */  in amdgpu_device_reset_sriov()
4080 	/* we need recover gart prior to run SMC/CP/SDMA resume */  in amdgpu_device_reset_sriov()
4081 	amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));  in amdgpu_device_reset_sriov()
4087 	/* now we are okay to resume SMC/CP/SDMA */  in amdgpu_device_reset_sriov()
4098 	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {  in amdgpu_device_reset_sriov()
4107  * amdgpu_device_has_job_running - check if there is any job in mirror list
4119 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_device_has_job_running()
4121 		if (!ring || !ring->sched.thread)  in amdgpu_device_has_job_running()
4124 		spin_lock(&ring->sched.job_list_lock);  in amdgpu_device_has_job_running()
4125 		job = list_first_entry_or_null(&ring->sched.ring_mirror_list,  in amdgpu_device_has_job_running()
4127 		spin_unlock(&ring->sched.job_list_lock);  in amdgpu_device_has_job_running()
4135  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4145 		dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");  in amdgpu_device_should_recover_gpu()
4155 	if (amdgpu_gpu_recovery == -1) {  in amdgpu_device_should_recover_gpu()
4156 		switch (adev->asic_type) {  in amdgpu_device_should_recover_gpu()
4185 		dev_info(adev->dev, "GPU recovery disabled.\n");  in amdgpu_device_should_recover_gpu()
4206 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_device_pre_asic_reset()
4208 		if (!ring || !ring->sched.thread)  in amdgpu_device_pre_asic_reset()
4216 		drm_sched_increase_karma(&job->base);  in amdgpu_device_pre_asic_reset()
4229 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");  in amdgpu_device_pre_asic_reset()
4259 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_do_asic_reset()
4260 				if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))  in amdgpu_do_asic_reset()
4261 					r = -EALREADY;  in amdgpu_do_asic_reset()
4266 				dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",  in amdgpu_do_asic_reset()
4267 					 r, adev_to_drm(tmp_adev)->unique);  in amdgpu_do_asic_reset()
4276 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_do_asic_reset()
4277 					flush_work(&tmp_adev->xgmi_reset_work);  in amdgpu_do_asic_reset()
4278 					r = tmp_adev->asic_reset_res;  in amdgpu_do_asic_reset()
4288 			if (tmp_adev->mmhub.funcs &&  in amdgpu_do_asic_reset()
4289 			    tmp_adev->mmhub.funcs->reset_ras_error_count)  in amdgpu_do_asic_reset()
4290 				tmp_adev->mmhub.funcs->reset_ras_error_count(tmp_adev);  in amdgpu_do_asic_reset()
4300 				dev_warn(tmp_adev->dev, "asic atom init failed!");  in amdgpu_do_asic_reset()
4303 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");  in amdgpu_do_asic_reset()
4314 				r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));  in amdgpu_do_asic_reset()
4355 					r = -EINVAL;  in amdgpu_do_asic_reset()
4360 				if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)  in amdgpu_do_asic_reset()
4370 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);  in amdgpu_do_asic_reset()
4373 				r = -EAGAIN;  in amdgpu_do_asic_reset()
4381 			tmp_adev->asic_reset_res = r;  in amdgpu_do_asic_reset()
4392 	if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)  in amdgpu_device_lock_adev()
4396 		down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);  in amdgpu_device_lock_adev()
4398 		down_write(&adev->reset_sem);  in amdgpu_device_lock_adev()
4401 	atomic_inc(&adev->gpu_reset_counter);  in amdgpu_device_lock_adev()
4404 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;  in amdgpu_device_lock_adev()
4407 		adev->mp1_state = PP_MP1_STATE_RESET;  in amdgpu_device_lock_adev()
4410 		adev->mp1_state = PP_MP1_STATE_NONE;  in amdgpu_device_lock_adev()
4420 	adev->mp1_state = PP_MP1_STATE_NONE;  in amdgpu_device_unlock_adev()
4421 	atomic_set(&adev->in_gpu_reset, 0);  in amdgpu_device_unlock_adev()
4422 	up_write(&adev->reset_sem);  in amdgpu_device_unlock_adev()
4429 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),  in amdgpu_device_resume_display_audio()
4430 			adev->pdev->bus->number, 1);  in amdgpu_device_resume_display_audio()
4432 		pm_runtime_enable(&(p->dev));  in amdgpu_device_resume_display_audio()
4433 		pm_runtime_resume(&(p->dev));  in amdgpu_device_resume_display_audio()
4450 		return -EINVAL;  in amdgpu_device_suspend_display_audio()
4452 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),  in amdgpu_device_suspend_display_audio()
4453 			adev->pdev->bus->number, 1);  in amdgpu_device_suspend_display_audio()
4455 		return -ENODEV;  in amdgpu_device_suspend_display_audio()
4457 	expires = pm_runtime_autosuspend_expiration(&(p->dev));  in amdgpu_device_suspend_display_audio()
4467 	while (!pm_runtime_status_suspended(&(p->dev))) {  in amdgpu_device_suspend_display_audio()
4468 		if (!pm_runtime_suspend(&(p->dev)))  in amdgpu_device_suspend_display_audio()
4472 			dev_warn(adev->dev, "failed to suspend display audio\n");  in amdgpu_device_suspend_display_audio()
4474 			return -ETIMEDOUT;  in amdgpu_device_suspend_display_audio()
4478 	pm_runtime_disable(&(p->dev));  in amdgpu_device_suspend_display_audio()
4484  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4490  * Attempt to do soft-reset or full-reset and reinitialize Asic
4515 	if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {  in amdgpu_device_gpu_recover()
4522 	dev_info(adev->dev, "GPU %s begin!\n",  in amdgpu_device_gpu_recover()
4534 		if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {  in amdgpu_device_gpu_recover()
4536 				job ? job->base.id : -1, hive->hive_id);  in amdgpu_device_gpu_recover()
4540 		mutex_lock(&hive->hive_lock);  in amdgpu_device_gpu_recover()
4549 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_device_gpu_recover()
4551 			return -ENODEV;  in amdgpu_device_gpu_recover()
4552 		if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list))  in amdgpu_device_gpu_recover()
4553 			list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list);  in amdgpu_device_gpu_recover()
4554 		device_list_handle = &hive->device_list;  in amdgpu_device_gpu_recover()
4556 		list_add_tail(&adev->gmc.xgmi.head, &device_list);  in amdgpu_device_gpu_recover()
4563 			dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",  in amdgpu_device_gpu_recover()
4564 				  job ? job->base.id : -1);  in amdgpu_device_gpu_recover()
4584 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);  in amdgpu_device_gpu_recover()
4603 			struct amdgpu_ring *ring = tmp_adev->rings[i];  in amdgpu_device_gpu_recover()
4605 			if (!ring || !ring->sched.thread)  in amdgpu_device_gpu_recover()
4608 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);  in amdgpu_device_gpu_recover()
4611 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);  in amdgpu_device_gpu_recover()
4622 	 * job->base holds a reference to parent fence  in amdgpu_device_gpu_recover()
4624 	if (job && job->base.s_fence->parent &&  in amdgpu_device_gpu_recover()
4625 	    dma_fence_is_signaled(job->base.s_fence->parent)) {  in amdgpu_device_gpu_recover()
4627 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");  in amdgpu_device_gpu_recover()
4638 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",  in amdgpu_device_gpu_recover()
4639 				  r, adev_to_drm(tmp_adev)->unique);  in amdgpu_device_gpu_recover()
4640 			tmp_adev->asic_reset_res = r;  in amdgpu_device_gpu_recover()
4649 			adev->asic_reset_res = r;  in amdgpu_device_gpu_recover()
4652 		if (r && r == -EAGAIN)  in amdgpu_device_gpu_recover()
4662 			struct amdgpu_ring *ring = tmp_adev->rings[i];  in amdgpu_device_gpu_recover()
4664 			if (!ring || !ring->sched.thread)  in amdgpu_device_gpu_recover()
4668 			if (!tmp_adev->asic_reset_res && !job_signaled)  in amdgpu_device_gpu_recover()
4669 				drm_sched_resubmit_jobs(&ring->sched);  in amdgpu_device_gpu_recover()
4671 			drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);  in amdgpu_device_gpu_recover()
4678 		tmp_adev->asic_reset_res = 0;  in amdgpu_device_gpu_recover()
4682 			dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));  in amdgpu_device_gpu_recover()
4685 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));  in amdgpu_device_gpu_recover()
4701 		atomic_set(&hive->in_reset, 0);  in amdgpu_device_gpu_recover()
4702 		mutex_unlock(&hive->hive_lock);  in amdgpu_device_gpu_recover()
4707 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);  in amdgpu_device_gpu_recover()
4712  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
4727 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;  in amdgpu_device_get_pcie_info()
4730 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;  in amdgpu_device_get_pcie_info()
4733 	if (pci_is_root_bus(adev->pdev->bus)) {  in amdgpu_device_get_pcie_info()
4734 		if (adev->pm.pcie_gen_mask == 0)  in amdgpu_device_get_pcie_info()
4735 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;  in amdgpu_device_get_pcie_info()
4736 		if (adev->pm.pcie_mlw_mask == 0)  in amdgpu_device_get_pcie_info()
4737 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;  in amdgpu_device_get_pcie_info()
4741 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)  in amdgpu_device_get_pcie_info()
4744 	pcie_bandwidth_available(adev->pdev, NULL,  in amdgpu_device_get_pcie_info()
4747 	if (adev->pm.pcie_gen_mask == 0) {  in amdgpu_device_get_pcie_info()
4749 		pdev = adev->pdev;  in amdgpu_device_get_pcie_info()
4752 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4757 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4762 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4766 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4769 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;  in amdgpu_device_get_pcie_info()
4773 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4777 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4782 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4786 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |  in amdgpu_device_get_pcie_info()
4789 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;  in amdgpu_device_get_pcie_info()
4793 	if (adev->pm.pcie_mlw_mask == 0) {  in amdgpu_device_get_pcie_info()
4795 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;  in amdgpu_device_get_pcie_info()
4799 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |  in amdgpu_device_get_pcie_info()
4808 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |  in amdgpu_device_get_pcie_info()
4816 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |  in amdgpu_device_get_pcie_info()
4823 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |  in amdgpu_device_get_pcie_info()
4829 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |  in amdgpu_device_get_pcie_info()
4834 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |  in amdgpu_device_get_pcie_info()
4838 				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;  in amdgpu_device_get_pcie_info()
4853 		return -ENOTSUPP;  in amdgpu_device_baco_enter()
4855 	if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)  in amdgpu_device_baco_enter()
4856 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);  in amdgpu_device_baco_enter()
4868 		return -ENOTSUPP;  in amdgpu_device_baco_exit()
4874 	if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)  in amdgpu_device_baco_exit()
4875 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);  in amdgpu_device_baco_exit()
4885 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_cancel_all_tdr()
4887 		if (!ring || !ring->sched.thread)  in amdgpu_cancel_all_tdr()
4890 		cancel_delayed_work_sync(&ring->sched.work_tdr);  in amdgpu_cancel_all_tdr()
4895  * amdgpu_pci_error_detected - Called when a PCI error is detected.
4911 	if (adev->gmc.xgmi.num_physical_nodes > 1) {  in amdgpu_pci_error_detected()
4923 		 * set  adev->in_gpu_reset in amdgpu_device_lock_adev  in amdgpu_pci_error_detected()
4925 		 * Locking adev->reset_sem will prevent any external access  in amdgpu_pci_error_detected()
4936 			struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_pci_error_detected()
4938 			if (!ring || !ring->sched.thread)  in amdgpu_pci_error_detected()
4941 			drm_sched_stop(&ring->sched, NULL);  in amdgpu_pci_error_detected()
4953  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
4961 	/* TODO - dump whatever for debugging purposes */  in amdgpu_pci_mmio_enabled()
4972  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
4977  * should resume normal operations.
4991 	list_add_tail(&adev->gmc.xgmi.head, &device_list);  in amdgpu_pci_slot_reset()
5000 	for (i = 0; i < adev->usec_timeout; i++) {  in amdgpu_pci_slot_reset()
5008 		r = -ETIME;  in amdgpu_pci_slot_reset()
5012 	adev->in_pci_err_recovery = true;	  in amdgpu_pci_slot_reset()
5014 	adev->in_pci_err_recovery = false;  in amdgpu_pci_slot_reset()
5022 		if (amdgpu_device_cache_pci_state(adev->pdev))  in amdgpu_pci_slot_reset()
5023 			pci_restore_state(adev->pdev);  in amdgpu_pci_slot_reset()
5035  * amdgpu_pci_resume() - resume normal ops after PCI reset
5039  * OK to resume normal operation. Use completion to allow
5040  * halted scsi ops to resume.
5049 	DRM_INFO("PCI error: resume callback!!\n");  in amdgpu_pci_resume()
5052 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_pci_resume()
5054 		if (!ring || !ring->sched.thread)  in amdgpu_pci_resume()
5058 		drm_sched_resubmit_jobs(&ring->sched);  in amdgpu_pci_resume()
5059 		drm_sched_start(&ring->sched, true);  in amdgpu_pci_resume()
5073 		kfree(adev->pci_state);  in amdgpu_device_cache_pci_state()
5075 		adev->pci_state = pci_store_saved_state(pdev);  in amdgpu_device_cache_pci_state()
5077 		if (!adev->pci_state) {  in amdgpu_device_cache_pci_state()
5095 	if (!adev->pci_state)  in amdgpu_device_load_pci_state()
5098 	r = pci_load_saved_state(pdev, adev->pci_state);  in amdgpu_device_load_pci_state()