Lines Matching +full:zap +full:- +full:shader
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
10 #include <linux/nvmem-consumer.h>
26 if (a5xx_gpu->has_whereami) { in update_shadow_rptr()
48 spin_lock_irqsave(&ring->preempt_lock, flags); in a5xx_flush()
51 ring->cur = ring->next; in a5xx_flush()
56 spin_unlock_irqrestore(&ring->preempt_lock, flags); in a5xx_flush()
62 if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu)) in a5xx_flush()
68 struct msm_ringbuffer *ring = submit->ring; in a5xx_submit_in_rb()
73 for (i = 0; i < submit->nr_cmds; i++) { in a5xx_submit_in_rb()
74 switch (submit->cmd[i].type) { in a5xx_submit_in_rb()
78 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) in a5xx_submit_in_rb()
83 obj = submit->bos[submit->cmd[i].idx].obj; in a5xx_submit_in_rb()
84 dwords = submit->cmd[i].size; in a5xx_submit_in_rb()
120 ring->memptrs->fence = submit->seqno; in a5xx_submit_in_rb()
128 struct msm_ringbuffer *ring = submit->ring; in a5xx_submit()
131 if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) { in a5xx_submit()
132 gpu->cur_ctx_seqno = 0; in a5xx_submit()
146 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); in a5xx_submit()
147 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id])); in a5xx_submit()
162 for (i = 0; i < submit->nr_cmds; i++) { in a5xx_submit()
163 switch (submit->cmd[i].type) { in a5xx_submit()
167 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno) in a5xx_submit()
172 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); in a5xx_submit()
173 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); in a5xx_submit()
174 OUT_RING(ring, submit->cmd[i].size); in a5xx_submit()
180 * Periodically update shadow-wptr if needed, so that we in a5xx_submit()
192 * are done rendering - otherwise a lucky preemption would start in a5xx_submit()
208 OUT_RING(ring, submit->seqno); in a5xx_submit()
219 OUT_RING(ring, submit->seqno); in a5xx_submit()
230 /* Data value - not used if the address above is 0 */ in a5xx_submit()
469 struct msm_ringbuffer *ring = gpu->rb[0]; in a5xx_me_init()
488 * Force a WFI after every direct-render 3D mode draw and every in a5xx_me_init()
504 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; in a5xx_me_init()
511 struct msm_ringbuffer *ring = gpu->rb[0]; in a5xx_preempt_start()
513 if (gpu->nr_rings == 1) in a5xx_preempt_start()
522 OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id])); in a5xx_preempt_start()
523 OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id])); in a5xx_preempt_start()
548 return a5xx_idle(gpu, ring) ? 0 : -EINVAL; in a5xx_preempt_start()
565 a5xx_gpu->has_whereami = true; in a5xx_ucode_check_version()
576 if (!a5xx_gpu->pm4_bo) { in a5xx_ucode_load()
577 a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu, in a5xx_ucode_load()
578 adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova); in a5xx_ucode_load()
581 if (IS_ERR(a5xx_gpu->pm4_bo)) { in a5xx_ucode_load()
582 ret = PTR_ERR(a5xx_gpu->pm4_bo); in a5xx_ucode_load()
583 a5xx_gpu->pm4_bo = NULL; in a5xx_ucode_load()
584 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n", in a5xx_ucode_load()
589 msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw"); in a5xx_ucode_load()
592 if (!a5xx_gpu->pfp_bo) { in a5xx_ucode_load()
593 a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu, in a5xx_ucode_load()
594 adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova); in a5xx_ucode_load()
596 if (IS_ERR(a5xx_gpu->pfp_bo)) { in a5xx_ucode_load()
597 ret = PTR_ERR(a5xx_gpu->pfp_bo); in a5xx_ucode_load()
598 a5xx_gpu->pfp_bo = NULL; in a5xx_ucode_load()
599 DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n", in a5xx_ucode_load()
604 msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw"); in a5xx_ucode_load()
605 a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo); in a5xx_ucode_load()
608 if (a5xx_gpu->has_whereami) { in a5xx_ucode_load()
609 if (!a5xx_gpu->shadow_bo) { in a5xx_ucode_load()
610 a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, in a5xx_ucode_load()
611 sizeof(u32) * gpu->nr_rings, in a5xx_ucode_load()
613 gpu->aspace, &a5xx_gpu->shadow_bo, in a5xx_ucode_load()
614 &a5xx_gpu->shadow_iova); in a5xx_ucode_load()
616 if (IS_ERR(a5xx_gpu->shadow)) in a5xx_ucode_load()
617 return PTR_ERR(a5xx_gpu->shadow); in a5xx_ucode_load()
619 msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow"); in a5xx_ucode_load()
621 } else if (gpu->nr_rings > 1) { in a5xx_ucode_load()
624 gpu->nr_rings = 1; in a5xx_ucode_load()
639 * to resume zap shader in a5xx_zap_shader_resume()
646 DRM_ERROR("%s: zap-shader resume failed: %d\n", in a5xx_zap_shader_resume()
647 gpu->name, ret); in a5xx_zap_shader_resume()
658 * If the zap shader is already loaded into memory we just need to kick in a5xx_zap_shader_init()
702 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) { in a5xx_hw_init()
704 * Mask out the activity signals from RB1-3 to avoid false in a5xx_hw_init()
748 /* Set the GMEM VA range (0 to gpu->gmem) */ in a5xx_hw_init()
752 0x00100000 + adreno_gpu->info->gmem - 1); in a5xx_hw_init()
785 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI) in a5xx_hw_init()
790 * for 1-SP GPUs, as it is enabled by default. in a5xx_hw_init()
811 * bug is fixed in latest A510 revision. To enable this bug fix - in a5xx_hw_init()
823 BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13); in a5xx_hw_init()
824 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13; in a5xx_hw_init()
877 * Disable the trusted memory range - we don't actually supported secure in a5xx_hw_init()
903 if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) { in a5xx_hw_init()
915 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova); in a5xx_hw_init()
916 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova); in a5xx_hw_init()
919 gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova); in a5xx_hw_init()
931 if (a5xx_gpu->shadow_bo) { in a5xx_hw_init()
933 shadowptr(a5xx_gpu, gpu->rb[0])); in a5xx_hw_init()
956 OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1); in a5xx_hw_init()
957 OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT)); in a5xx_hw_init()
959 a5xx_flush(gpu, gpu->rb[0], true); in a5xx_hw_init()
960 if (!a5xx_idle(gpu, gpu->rb[0])) in a5xx_hw_init()
961 return -EINVAL; in a5xx_hw_init()
966 * try to load a zap shader into the secure world. If successful in a5xx_hw_init()
974 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); in a5xx_hw_init()
975 OUT_RING(gpu->rb[0], 0x00000000); in a5xx_hw_init()
977 a5xx_flush(gpu, gpu->rb[0], true); in a5xx_hw_init()
978 if (!a5xx_idle(gpu, gpu->rb[0])) in a5xx_hw_init()
979 return -EINVAL; in a5xx_hw_init()
980 } else if (ret == -ENODEV) { in a5xx_hw_init()
982 * This device does not use zap shader (but print a warning in a5xx_hw_init()
987 dev_warn_once(gpu->dev->dev, in a5xx_hw_init()
988 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); in a5xx_hw_init()
994 /* Last step - yield the ringbuffer */ in a5xx_hw_init()
1025 DBG("%s", gpu->name); in a5xx_destroy()
1029 if (a5xx_gpu->pm4_bo) { in a5xx_destroy()
1030 msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace); in a5xx_destroy()
1031 drm_gem_object_put(a5xx_gpu->pm4_bo); in a5xx_destroy()
1034 if (a5xx_gpu->pfp_bo) { in a5xx_destroy()
1035 msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace); in a5xx_destroy()
1036 drm_gem_object_put(a5xx_gpu->pfp_bo); in a5xx_destroy()
1039 if (a5xx_gpu->gpmu_bo) { in a5xx_destroy()
1040 msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace); in a5xx_destroy()
1041 drm_gem_object_put(a5xx_gpu->gpmu_bo); in a5xx_destroy()
1044 if (a5xx_gpu->shadow_bo) { in a5xx_destroy()
1045 msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace); in a5xx_destroy()
1046 drm_gem_object_put(a5xx_gpu->shadow_bo); in a5xx_destroy()
1071 if (ring != a5xx_gpu->cur_ring) { in a5xx_idle()
1072 WARN(1, "Tried to idle a non-current ringbuffer\n"); in a5xx_idle()
1082 gpu->name, __builtin_return_address(0), in a5xx_idle()
1106 snprintf(block, sizeof(block), "%x", info->fsynr1); in a5xx_fault_handler()
1128 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n", in a5xx_cp_err_irq()
1133 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n", in a5xx_cp_err_irq()
1137 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n"); in a5xx_cp_err_irq()
1142 dev_err_ratelimited(gpu->dev->dev, in a5xx_cp_err_irq()
1155 dev_err_ratelimited(gpu->dev->dev, in a5xx_cp_err_irq()
1167 dev_err_ratelimited(gpu->dev->dev, in a5xx_rbbm_err_irq()
1182 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n"); in a5xx_rbbm_err_irq()
1185 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n", in a5xx_rbbm_err_irq()
1189 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n", in a5xx_rbbm_err_irq()
1193 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n", in a5xx_rbbm_err_irq()
1197 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n"); in a5xx_rbbm_err_irq()
1200 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n"); in a5xx_rbbm_err_irq()
1209 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n", in a5xx_uche_err_irq()
1215 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n"); in a5xx_gpmu_err_irq()
1220 struct drm_device *dev = gpu->dev; in a5xx_fault_detect_irq()
1221 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); in a5xx_fault_detect_irq()
1232 …DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4… in a5xx_fault_detect_irq()
1233 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, in a5xx_fault_detect_irq()
1243 del_timer(&gpu->hangcheck_timer); in a5xx_fault_detect_irq()
1245 kthread_queue_work(gpu->worker, &gpu->recover_work); in a5xx_fault_detect_irq()
1258 struct msm_drm_private *priv = gpu->dev->dev_private; in a5xx_irq()
1262 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it in a5xx_irq()
1268 if (priv->disable_err_irq) { in a5xx_irq()
1333 DRM_DEV_INFO(gpu->dev->dev, "status: %08x\n", in a5xx_dump()
1368 gpu->name, in a5xx_pm_resume()
1379 gpu->name); in a5xx_pm_resume()
1416 if (a5xx_gpu->has_whereami) in a5xx_pm_suspend()
1417 for (i = 0; i < gpu->nr_rings; i++) in a5xx_pm_suspend()
1418 a5xx_gpu->shadow[i] = 0; in a5xx_pm_suspend()
1444 dumper->ptr = msm_gem_kernel_new(gpu->dev, in a5xx_crashdumper_init()
1445 SZ_1M, MSM_BO_WC, gpu->aspace, in a5xx_crashdumper_init()
1446 &dumper->bo, &dumper->iova); in a5xx_crashdumper_init()
1448 if (!IS_ERR(dumper->ptr)) in a5xx_crashdumper_init()
1449 msm_gem_object_set_name(dumper->bo, "crashdump"); in a5xx_crashdumper_init()
1451 return PTR_ERR_OR_ZERO(dumper->ptr); in a5xx_crashdumper_init()
1459 if (IS_ERR_OR_NULL(dumper->ptr)) in a5xx_crashdumper_run()
1460 return -EINVAL; in a5xx_crashdumper_run()
1462 gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova); in a5xx_crashdumper_run()
1480 { 0x35, 0xe00, 0x32 }, /* HSLQ non-context */
1485 { 0x3f, 0x0ec0, 0x40 }, /* SP non-context */
1490 { 0x3a, 0x0f00, 0x1c }, /* TP non-context */
1518 a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL); in a5xx_gpu_state_get_hlsq_regs()
1519 if (!a5xx_state->hlsqregs) in a5xx_gpu_state_get_hlsq_regs()
1544 kfree(a5xx_state->hlsqregs); in a5xx_gpu_state_get_hlsq_regs()
1545 msm_gem_kernel_put(dumper.bo, gpu->aspace); in a5xx_gpu_state_get_hlsq_regs()
1550 memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K), in a5xx_gpu_state_get_hlsq_regs()
1553 msm_gem_kernel_put(dumper.bo, gpu->aspace); in a5xx_gpu_state_get_hlsq_regs()
1563 return ERR_PTR(-ENOMEM); in a5xx_gpu_state_get()
1569 adreno_gpu_state_get(gpu, &(a5xx_state->base)); in a5xx_gpu_state_get()
1571 a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS); in a5xx_gpu_state_get()
1583 return &a5xx_state->base; in a5xx_gpu_state_get()
1593 kfree(a5xx_state->hlsqregs); in a5xx_gpu_state_destroy()
1604 return kref_put(&state->ref, a5xx_gpu_state_destroy); in a5xx_gpu_state_put()
1623 if (!a5xx_state->hlsqregs) in a5xx_show()
1626 drm_printf(p, "registers-hlsq:\n"); in a5xx_show()
1640 if (a5xx_state->hlsqregs[pos] == 0xdeadbeef) in a5xx_show()
1643 drm_printf(p, " - { offset: 0x%04x, value: 0x%08x }\n", in a5xx_show()
1644 o << 2, a5xx_state->hlsqregs[pos]); in a5xx_show()
1655 return a5xx_gpu->cur_ring; in a5xx_active_ring()
1663 *out_sample_rate = clk_get_rate(gpu->core_clk); in a5xx_gpu_busy()
1673 if (a5xx_gpu->has_whereami) in a5xx_get_rptr()
1674 return a5xx_gpu->shadow[ring->id]; in a5xx_get_rptr()
1676 return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR); in a5xx_get_rptr()
1713 * If the OPP table specifies a opp-supported-hw property then we have in check_speed_bin()
1741 struct msm_drm_private *priv = dev->dev_private; in a5xx_gpu_init()
1742 struct platform_device *pdev = priv->gpu_pdev; in a5xx_gpu_init()
1743 struct adreno_platform_config *config = pdev->dev.platform_data; in a5xx_gpu_init()
1751 DRM_DEV_ERROR(dev->dev, "No A5XX device is defined\n"); in a5xx_gpu_init()
1752 return ERR_PTR(-ENXIO); in a5xx_gpu_init()
1757 return ERR_PTR(-ENOMEM); in a5xx_gpu_init()
1759 adreno_gpu = &a5xx_gpu->base; in a5xx_gpu_init()
1760 gpu = &adreno_gpu->base; in a5xx_gpu_init()
1762 adreno_gpu->registers = a5xx_registers; in a5xx_gpu_init()
1764 a5xx_gpu->lm_leakage = 0x4E001A; in a5xx_gpu_init()
1766 check_speed_bin(&pdev->dev); in a5xx_gpu_init()
1770 if (config->info->revn == 510) in a5xx_gpu_init()
1775 a5xx_destroy(&(a5xx_gpu->base.base)); in a5xx_gpu_init()
1779 if (gpu->aspace) in a5xx_gpu_init()
1780 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler); in a5xx_gpu_init()
1787 adreno_gpu->ubwc_config.highest_bank_bit = 15; in a5xx_gpu_init()
1789 adreno_gpu->ubwc_config.highest_bank_bit = 14; in a5xx_gpu_init()