255 		return "MMU bypass is disallowed for this StreamID";  in ivpu_mmu_event_to_str()
313 		ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
317 		ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);  in ivpu_mmu_config_check()
321 		ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);  in ivpu_mmu_config_check()
332 		ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
337 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cdtab_alloc()  local
338 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_cdtab_alloc()
345 	ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);  in ivpu_mmu_cdtab_alloc()
352 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_alloc()  local
353 	struct ivpu_mmu_strtab *strtab = &mmu->strtab;  in ivpu_mmu_strtab_alloc()
364 	ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",  in ivpu_mmu_strtab_alloc()
372 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cmdq_alloc()  local
373 	struct ivpu_mmu_queue *q = &mmu->cmdq;  in ivpu_mmu_cmdq_alloc()
383 	ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_cmdq_alloc()
391 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_evtq_alloc()  local
392 	struct ivpu_mmu_queue *q = &mmu->evtq;  in ivpu_mmu_evtq_alloc()
402 	ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_evtq_alloc()
465 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_wait_for_cons()
492 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_cmd_write()
497 		ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);  in ivpu_mmu_cmdq_cmd_write()
505 	ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);  in ivpu_mmu_cmdq_cmd_write()
512 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_sync()
532 		ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,  in ivpu_mmu_cmdq_sync()
565 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_reset()  local
569 	memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);  in ivpu_mmu_reset()
570 	clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);  in ivpu_mmu_reset()
571 	mmu->cmdq.prod = 0;  in ivpu_mmu_reset()
572 	mmu->cmdq.cons = 0;  in ivpu_mmu_reset()
574 	memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);  in ivpu_mmu_reset()
575 	mmu->evtq.prod = 0;  in ivpu_mmu_reset()
576 	mmu->evtq.cons = 0;  in ivpu_mmu_reset()
590 	REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q);  in ivpu_mmu_reset()
591 	REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg);  in ivpu_mmu_reset()
593 	REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q);  in ivpu_mmu_reset()
614 	REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q);  in ivpu_mmu_reset()
638 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_link_cd()  local
639 	struct ivpu_mmu_strtab *strtab = &mmu->strtab;  in ivpu_mmu_strtab_link_cd()
640 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_strtab_link_cd()
666 	ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);  in ivpu_mmu_strtab_link_cd()
679 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_invalidate_tlb()  local
682 	mutex_lock(&mmu->lock);  in ivpu_mmu_invalidate_tlb()
683 	if (!mmu->on)  in ivpu_mmu_invalidate_tlb()
692 	mutex_unlock(&mmu->lock);  in ivpu_mmu_invalidate_tlb()
698 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cd_add()  local
699 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_cd_add()
740 	ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",  in ivpu_mmu_cd_add()
743 	mutex_lock(&mmu->lock);  in ivpu_mmu_cd_add()
744 	if (!mmu->on)  in ivpu_mmu_cd_add()
753 	mutex_unlock(&mmu->lock);  in ivpu_mmu_cd_add()
786 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_init()  local
789 	ivpu_dbg(vdev, MMU, "Init..\n");  in ivpu_mmu_init()
793 	ret = drmm_mutex_init(&vdev->drm, &mmu->lock);  in ivpu_mmu_init()
815 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);  in ivpu_mmu_init()
819 	ivpu_dbg(vdev, MMU, "Init done\n");  in ivpu_mmu_init()
826 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_enable()  local
829 	mutex_lock(&mmu->lock);  in ivpu_mmu_enable()
831 	mmu->on = true;  in ivpu_mmu_enable()
835 		ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);  in ivpu_mmu_enable()
851 	mutex_unlock(&mmu->lock);  in ivpu_mmu_enable()
855 	mmu->on = false;  in ivpu_mmu_enable()
856 	mutex_unlock(&mmu->lock);  in ivpu_mmu_enable()
862 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_disable()  local
864 	mutex_lock(&mmu->lock);  in ivpu_mmu_disable()
865 	mmu->on = false;  in ivpu_mmu_disable()
866 	mutex_unlock(&mmu->lock);  in ivpu_mmu_disable()
877 …ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch…  in ivpu_mmu_dump_event()
883 	struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;  in ivpu_mmu_get_event()
900 	ivpu_dbg(vdev, IRQ, "MMU event queue\n");  in ivpu_mmu_irq_evtq_handler()
907 			ivpu_pm_trigger_recovery(vdev, "MMU event");  in ivpu_mmu_irq_evtq_handler()
912 		REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);  in ivpu_mmu_irq_evtq_handler()
928 	ivpu_dbg(vdev, IRQ, "MMU error\n");  in ivpu_mmu_irq_gerr_handler()
938 		ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
941 		ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
944 		ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
947 		ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
950 		ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
953 		ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
956 		ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()