17 static void a5xx_dump(struct msm_gpu *gpu);
21 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,  in a5xx_flush()  argument
24 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_flush()
54 		gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);  in a5xx_flush()
57 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)  in a5xx_submit_in_rb()  argument
59 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a5xx_submit_in_rb()
104 	a5xx_flush(gpu, ring, true);  in a5xx_submit_in_rb()
105 	a5xx_preempt_trigger(gpu);  in a5xx_submit_in_rb()
111 	a5xx_idle(gpu, ring);  in a5xx_submit_in_rb()
113 	msm_gpu_retire(gpu);  in a5xx_submit_in_rb()
116 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)  in a5xx_submit()  argument
118 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_submit()
120 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a5xx_submit()
126 		a5xx_submit_in_rb(gpu, submit);  in a5xx_submit()
219 	a5xx_flush(gpu, ring, false);  in a5xx_submit()
222 	a5xx_preempt_trigger(gpu);  in a5xx_submit()
323 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)  in a5xx_set_hwcg()  argument
325 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_set_hwcg()
329 		gpu_write(gpu, a5xx_hwcg[i].offset,  in a5xx_set_hwcg()
333 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);  in a5xx_set_hwcg()
334 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);  in a5xx_set_hwcg()
337 	gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);  in a5xx_set_hwcg()
338 	gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);  in a5xx_set_hwcg()
341 static int a5xx_me_init(struct msm_gpu *gpu)  in a5xx_me_init()  argument
343 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_me_init()
344 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_me_init()
378 	a5xx_flush(gpu, ring, true);  in a5xx_me_init()
379 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_me_init()
382 static int a5xx_preempt_start(struct msm_gpu *gpu)  in a5xx_preempt_start()  argument
384 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_start()
386 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_preempt_start()
388 	if (gpu->nr_rings == 1)  in a5xx_preempt_start()
421 	a5xx_flush(gpu, ring, false);  in a5xx_preempt_start()
423 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_preempt_start()
445 static int a5xx_ucode_init(struct msm_gpu *gpu)  in a5xx_ucode_init()  argument
447 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_ucode_init()
452 		a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_init()
459 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",  in a5xx_ucode_init()
468 		a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_init()
474 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",  in a5xx_ucode_init()
483 	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,  in a5xx_ucode_init()
486 	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,  in a5xx_ucode_init()
494 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)  in a5xx_zap_shader_resume()  argument
501 			gpu->name, ret);  in a5xx_zap_shader_resume()
506 static int a5xx_zap_shader_init(struct msm_gpu *gpu)  in a5xx_zap_shader_init()  argument
516 		return a5xx_zap_shader_resume(gpu);  in a5xx_zap_shader_init()
518 	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);  in a5xx_zap_shader_init()
537 static int a5xx_hw_init(struct msm_gpu *gpu)  in a5xx_hw_init()  argument
539 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_hw_init()
543 	gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);  in a5xx_hw_init()
546 		gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);  in a5xx_hw_init()
548 	/* Make all blocks contribute to the GPU BUSY perf counter */  in a5xx_hw_init()
549 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);  in a5xx_hw_init()
552 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);  in a5xx_hw_init()
560 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,  in a5xx_hw_init()
562 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,  in a5xx_hw_init()
564 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,  in a5xx_hw_init()
566 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,  in a5xx_hw_init()
568 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,  in a5xx_hw_init()
570 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,  in a5xx_hw_init()
572 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,  in a5xx_hw_init()
574 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,  in a5xx_hw_init()
579 	gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,  in a5xx_hw_init()
583 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);  in a5xx_hw_init()
586 	gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);  in a5xx_hw_init()
589 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);  in a5xx_hw_init()
592 	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);  in a5xx_hw_init()
595 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);  in a5xx_hw_init()
596 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);  in a5xx_hw_init()
597 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);  in a5xx_hw_init()
598 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);  in a5xx_hw_init()
600 	/* Set the GMEM VA range (0 to gpu->gmem) */  in a5xx_hw_init()
601 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);  in a5xx_hw_init()
602 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);  in a5xx_hw_init()
603 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,  in a5xx_hw_init()
605 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);  in a5xx_hw_init()
608 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);  in a5xx_hw_init()
609 		gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);  in a5xx_hw_init()
610 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);  in a5xx_hw_init()
611 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);  in a5xx_hw_init()
612 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,  in a5xx_hw_init()
615 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);  in a5xx_hw_init()
617 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);  in a5xx_hw_init()
619 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);  in a5xx_hw_init()
620 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);  in a5xx_hw_init()
621 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);  in a5xx_hw_init()
622 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,  in a5xx_hw_init()
627 		gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));  in a5xx_hw_init()
629 	gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);  in a5xx_hw_init()
632 	gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);  in a5xx_hw_init()
635 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);  in a5xx_hw_init()
642 	 *  CCU to be interpreted differently. This can cause gpu fault. This  in a5xx_hw_init()
648 		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);  in a5xx_hw_init()
651 	a5xx_set_hwcg(gpu, true);  in a5xx_hw_init()
653 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);  in a5xx_hw_init()
656 	gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);  in a5xx_hw_init()
657 	gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);  in a5xx_hw_init()
659 		gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, 2);  in a5xx_hw_init()
662 	gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);  in a5xx_hw_init()
665 	gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));  in a5xx_hw_init()
666 	gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));  in a5xx_hw_init()
667 	gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));  in a5xx_hw_init()
668 	gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));  in a5xx_hw_init()
669 	gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));  in a5xx_hw_init()
670 	gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));  in a5xx_hw_init()
673 	gpu_write(gpu, REG_A5XX_CP_PROTECT(6),  in a5xx_hw_init()
676 	gpu_write(gpu, REG_A5XX_CP_PROTECT(7),  in a5xx_hw_init()
680 	gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));  in a5xx_hw_init()
681 	gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));  in a5xx_hw_init()
682 	gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));  in a5xx_hw_init()
683 	gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));  in a5xx_hw_init()
686 	gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));  in a5xx_hw_init()
687 	gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));  in a5xx_hw_init()
690 	gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));  in a5xx_hw_init()
691 	gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));  in a5xx_hw_init()
694 	gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));  in a5xx_hw_init()
697 		gpu_write(gpu, REG_A5XX_CP_PROTECT(17),  in a5xx_hw_init()
700 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);  in a5xx_hw_init()
706 	gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,  in a5xx_hw_init()
708 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);  in a5xx_hw_init()
710 	/* Put the GPU into 64 bit by default */  in a5xx_hw_init()
711 	gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
712 	gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
713 	gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
714 	gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
715 	gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
716 	gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
717 	gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
718 	gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
719 	gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
720 	gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
721 	gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
722 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
730 		gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));  in a5xx_hw_init()
731 		gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);  in a5xx_hw_init()
734 	ret = adreno_hw_init(gpu);  in a5xx_hw_init()
739 		a5xx_gpmu_ucode_init(gpu);  in a5xx_hw_init()
741 	ret = a5xx_ucode_init(gpu);  in a5xx_hw_init()
746 	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, REG_A5XX_CP_RB_BASE_HI,  in a5xx_hw_init()
747 		gpu->rb[0]->iova);  in a5xx_hw_init()
755 	gpu_write(gpu, REG_A5XX_CP_RB_CNTL,  in a5xx_hw_init()
759 	if (!a5xx_gpu->has_whereami && gpu->nr_rings > 1) {  in a5xx_hw_init()
760 		a5xx_preempt_fini(gpu);  in a5xx_hw_init()
761 		gpu->nr_rings = 1;  in a5xx_hw_init()
765 			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,  in a5xx_hw_init()
766 				sizeof(u32) * gpu->nr_rings,  in a5xx_hw_init()
768 				gpu->aspace, &a5xx_gpu->shadow_bo,  in a5xx_hw_init()
775 		gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,  in a5xx_hw_init()
776 			REG_A5XX_CP_RB_RPTR_ADDR_HI, shadowptr(a5xx_gpu, gpu->rb[0]));  in a5xx_hw_init()
779 	a5xx_preempt_hw_init(gpu);  in a5xx_hw_init()
782 	gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);  in a5xx_hw_init()
785 	gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);  in a5xx_hw_init()
786 	ret = a5xx_me_init(gpu);  in a5xx_hw_init()
790 	ret = a5xx_power_init(gpu);  in a5xx_hw_init()
799 		OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);  in a5xx_hw_init()
800 		OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));  in a5xx_hw_init()
802 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
803 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
815 	ret = a5xx_zap_shader_init(gpu);  in a5xx_hw_init()
817 		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);  in a5xx_hw_init()
818 		OUT_RING(gpu->rb[0], 0x00000000);  in a5xx_hw_init()
820 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
821 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
830 		dev_warn_once(gpu->dev->dev,  in a5xx_hw_init()
832 		gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);  in a5xx_hw_init()
838 	a5xx_preempt_start(gpu);  in a5xx_hw_init()
843 static void a5xx_recover(struct msm_gpu *gpu)  in a5xx_recover()  argument
847 	adreno_dump_info(gpu);  in a5xx_recover()
851 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));  in a5xx_recover()
855 		a5xx_dump(gpu);  in a5xx_recover()
857 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);  in a5xx_recover()
858 	gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);  in a5xx_recover()
859 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);  in a5xx_recover()
860 	adreno_recover(gpu);  in a5xx_recover()
863 static void a5xx_destroy(struct msm_gpu *gpu)  in a5xx_destroy()  argument
865 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_destroy()
868 	DBG("%s", gpu->name);  in a5xx_destroy()
870 	a5xx_preempt_fini(gpu);  in a5xx_destroy()
873 		msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);  in a5xx_destroy()
878 		msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);  in a5xx_destroy()
883 		msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);  in a5xx_destroy()
888 		msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);  in a5xx_destroy()
896 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)  in _a5xx_check_idle()  argument
898 	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)  in _a5xx_check_idle()
902 	 * Nearly every abnormality ends up pausing the GPU and triggering a  in _a5xx_check_idle()
905 	return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &  in _a5xx_check_idle()
909 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a5xx_idle()  argument
911 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_idle()
920 	if (!adreno_idle(gpu, ring))  in a5xx_idle()
923 	if (spin_until(_a5xx_check_idle(gpu))) {  in a5xx_idle()
924 		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",  in a5xx_idle()
925 			gpu->name, __builtin_return_address(0),  in a5xx_idle()
926 			gpu_read(gpu, REG_A5XX_RBBM_STATUS),  in a5xx_idle()
927 			gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),  in a5xx_idle()
928 			gpu_read(gpu, REG_A5XX_CP_RB_RPTR),  in a5xx_idle()
929 			gpu_read(gpu, REG_A5XX_CP_RB_WPTR));  in a5xx_idle()
938 	struct msm_gpu *gpu = arg;  in a5xx_fault_handler()  local
939 	pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",  in a5xx_fault_handler()
941 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),  in a5xx_fault_handler()
942 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),  in a5xx_fault_handler()
943 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),  in a5xx_fault_handler()
944 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));  in a5xx_fault_handler()
949 static void a5xx_cp_err_irq(struct msm_gpu *gpu)  in a5xx_cp_err_irq()  argument
951 	u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);  in a5xx_cp_err_irq()
956 		gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);  in a5xx_cp_err_irq()
963 		gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);  in a5xx_cp_err_irq()
964 		val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);  in a5xx_cp_err_irq()
966 		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",  in a5xx_cp_err_irq()
971 		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",  in a5xx_cp_err_irq()
972 			gpu_read(gpu, REG_A5XX_CP_HW_FAULT));  in a5xx_cp_err_irq()
975 		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");  in a5xx_cp_err_irq()
978 		u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);  in a5xx_cp_err_irq()
980 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
987 		u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);  in a5xx_cp_err_irq()
993 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
1000 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)  in a5xx_rbbm_err_irq()  argument
1003 		u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);  in a5xx_rbbm_err_irq()
1005 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_rbbm_err_irq()
1012 		gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));  in a5xx_rbbm_err_irq()
1015 		gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,  in a5xx_rbbm_err_irq()
1020 		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");  in a5xx_rbbm_err_irq()
1023 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1024 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1027 		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1028 			gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1031 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1032 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1035 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");  in a5xx_rbbm_err_irq()
1038 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");  in a5xx_rbbm_err_irq()
1041 static void a5xx_uche_err_irq(struct msm_gpu *gpu)  in a5xx_uche_err_irq()  argument
1043 	uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);  in a5xx_uche_err_irq()
1045 	addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);  in a5xx_uche_err_irq()
1047 	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",  in a5xx_uche_err_irq()
1051 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)  in a5xx_gpmu_err_irq()  argument
1053 	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");  in a5xx_gpmu_err_irq()
1056 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)  in a5xx_fault_detect_irq()  argument
1058 	struct drm_device *dev = gpu->dev;  in a5xx_fault_detect_irq()
1060 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in a5xx_fault_detect_irq()
1062 …DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4…  in a5xx_fault_detect_irq()
1064 		gpu_read(gpu, REG_A5XX_RBBM_STATUS),  in a5xx_fault_detect_irq()
1065 		gpu_read(gpu, REG_A5XX_CP_RB_RPTR),  in a5xx_fault_detect_irq()
1066 		gpu_read(gpu, REG_A5XX_CP_RB_WPTR),  in a5xx_fault_detect_irq()
1067 		gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI),  in a5xx_fault_detect_irq()
1068 		gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),  in a5xx_fault_detect_irq()
1069 		gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI),  in a5xx_fault_detect_irq()
1070 		gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));  in a5xx_fault_detect_irq()
1073 	del_timer(&gpu->hangcheck_timer);  in a5xx_fault_detect_irq()
1075 	queue_work(priv->wq, &gpu->recover_work);  in a5xx_fault_detect_irq()
1086 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)  in a5xx_irq()  argument
1088 	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);  in a5xx_irq()
1094 	gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,  in a5xx_irq()
1099 		a5xx_rbbm_err_irq(gpu, status);  in a5xx_irq()
1102 		a5xx_cp_err_irq(gpu);  in a5xx_irq()
1105 		a5xx_fault_detect_irq(gpu);  in a5xx_irq()
1108 		a5xx_uche_err_irq(gpu);  in a5xx_irq()
1111 		a5xx_gpmu_err_irq(gpu);  in a5xx_irq()
1114 		a5xx_preempt_trigger(gpu);  in a5xx_irq()
1115 		msm_gpu_retire(gpu);  in a5xx_irq()
1119 		a5xx_preempt_irq(gpu);  in a5xx_irq()
1155 static void a5xx_dump(struct msm_gpu *gpu)  in a5xx_dump()  argument
1157 	DRM_DEV_INFO(gpu->dev->dev, "status:   %08x\n",  in a5xx_dump()
1158 		gpu_read(gpu, REG_A5XX_RBBM_STATUS));  in a5xx_dump()
1159 	adreno_dump(gpu);  in a5xx_dump()
1162 static int a5xx_pm_resume(struct msm_gpu *gpu)  in a5xx_pm_resume()  argument
1164 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_pm_resume()
1168 	ret = msm_gpu_pm_resume(gpu);  in a5xx_pm_resume()
1174 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);  in a5xx_pm_resume()
1175 		a5xx_set_hwcg(gpu, true);  in a5xx_pm_resume()
1177 		gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);  in a5xx_pm_resume()
1182 	gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);  in a5xx_pm_resume()
1187 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,  in a5xx_pm_resume()
1191 			gpu->name,  in a5xx_pm_resume()
1192 			gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));  in a5xx_pm_resume()
1197 	gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);  in a5xx_pm_resume()
1198 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,  in a5xx_pm_resume()
1202 			gpu->name);  in a5xx_pm_resume()
1207 static int a5xx_pm_suspend(struct msm_gpu *gpu)  in a5xx_pm_suspend()  argument
1209 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_pm_suspend()
1217 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);  in a5xx_pm_suspend()
1218 	spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &  in a5xx_pm_suspend()
1221 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);  in a5xx_pm_suspend()
1227 	gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);  in a5xx_pm_suspend()
1228 	gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);  in a5xx_pm_suspend()
1230 	return msm_gpu_pm_suspend(gpu);  in a5xx_pm_suspend()
1233 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)  in a5xx_get_timestamp()  argument
1235 	*value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,  in a5xx_get_timestamp()
1252 static int a5xx_crashdumper_init(struct msm_gpu *gpu,  in a5xx_crashdumper_init()  argument
1255 	dumper->ptr = msm_gem_kernel_new_locked(gpu->dev,  in a5xx_crashdumper_init()
1256 		SZ_1M, MSM_BO_UNCACHED, gpu->aspace,  in a5xx_crashdumper_init()
1265 static int a5xx_crashdumper_run(struct msm_gpu *gpu,  in a5xx_crashdumper_run()  argument
1273 	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO,  in a5xx_crashdumper_run()
1276 	gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);  in a5xx_crashdumper_run()
1278 	return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,  in a5xx_crashdumper_run()
1309 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,  in a5xx_gpu_state_get_hlsq_regs()  argument
1317 	if (a5xx_crashdumper_init(gpu, &dumper))  in a5xx_gpu_state_get_hlsq_regs()
1355 	if (a5xx_crashdumper_run(gpu, &dumper)) {  in a5xx_gpu_state_get_hlsq_regs()
1357 		msm_gem_kernel_put(dumper.bo, gpu->aspace, true);  in a5xx_gpu_state_get_hlsq_regs()
1365 	msm_gem_kernel_put(dumper.bo, gpu->aspace, true);  in a5xx_gpu_state_get_hlsq_regs()
1368 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)  in a5xx_gpu_state_get()  argument
1377 	a5xx_set_hwcg(gpu, false);  in a5xx_gpu_state_get()
1380 	adreno_gpu_state_get(gpu, &(a5xx_state->base));  in a5xx_gpu_state_get()
1382 	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);  in a5xx_gpu_state_get()
1385 	a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);  in a5xx_gpu_state_get()
1387 	a5xx_set_hwcg(gpu, true);  in a5xx_gpu_state_get()
1415 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,  in a5xx_show()  argument
1426 	adreno_show(gpu, state, p);  in a5xx_show()
1456 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)  in a5xx_active_ring()  argument
1458 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_active_ring()
1464 static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu)  in a5xx_gpu_busy()  argument
1468 	/* Only read the gpu busy if the hardware is already active */  in a5xx_gpu_busy()
1469 	if (pm_runtime_get_if_in_use(&gpu->pdev->dev) == 0)  in a5xx_gpu_busy()
1472 	busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,  in a5xx_gpu_busy()
1475 	busy_time = busy_cycles - gpu->devfreq.busy_cycles;  in a5xx_gpu_busy()
1476 	do_div(busy_time, clk_get_rate(gpu->core_clk) / 1000000);  in a5xx_gpu_busy()
1478 	gpu->devfreq.busy_cycles = busy_cycles;  in a5xx_gpu_busy()
1480 	pm_runtime_put(&gpu->pdev->dev);  in a5xx_gpu_busy()
1488 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a5xx_get_rptr()  argument
1490 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_get_rptr()
1496 	return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);  in a5xx_get_rptr()
1563 	struct msm_gpu *gpu;  in a5xx_gpu_init()  local
1576 	gpu = &adreno_gpu->base;  in a5xx_gpu_init()
1590 	if (gpu->aspace)  in a5xx_gpu_init()
1591 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);  in a5xx_gpu_init()
1594 	a5xx_preempt_init(gpu);  in a5xx_gpu_init()
1596 	return gpu;  in a5xx_gpu_init()