Lines Matching +full:disable +full:- +full:eop
71 struct amdgpu_device *adev = ring->adev; in mes_v11_0_ring_set_wptr()
73 if (ring->use_doorbell) { in mes_v11_0_ring_set_wptr()
74 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, in mes_v11_0_ring_set_wptr()
75 ring->wptr); in mes_v11_0_ring_set_wptr()
76 WDOORBELL64(ring->doorbell_index, ring->wptr); in mes_v11_0_ring_set_wptr()
84 return *ring->rptr_cpu_addr; in mes_v11_0_ring_get_rptr()
91 if (ring->use_doorbell) in mes_v11_0_ring_get_wptr()
92 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); in mes_v11_0_ring_get_wptr()
145 if (x_pkt->header.opcode < ARRAY_SIZE(mes_v11_0_opcodes)) in mes_v11_0_get_op_string()
146 op_str = mes_v11_0_opcodes[x_pkt->header.opcode]; in mes_v11_0_get_op_string()
155 if ((x_pkt->header.opcode == MES_SCH_API_MISC) && in mes_v11_0_get_misc_op_string()
156 (x_pkt->opcode < ARRAY_SIZE(mes_v11_0_misc_opcodes))) in mes_v11_0_get_misc_op_string()
157 op_str = mes_v11_0_misc_opcodes[x_pkt->opcode]; in mes_v11_0_get_misc_op_string()
168 struct amdgpu_device *adev = mes->adev; in mes_v11_0_submit_pkt_and_poll_completion()
169 struct amdgpu_ring *ring = &mes->ring[0]; in mes_v11_0_submit_pkt_and_poll_completion()
180 if (x_pkt->header.opcode >= MES_SCH_API_MAX) in mes_v11_0_submit_pkt_and_poll_completion()
181 return -EINVAL; in mes_v11_0_submit_pkt_and_poll_completion()
194 status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4); in mes_v11_0_submit_pkt_and_poll_completion()
195 status_ptr = (u64 *)&adev->wb.wb[status_offset]; in mes_v11_0_submit_pkt_and_poll_completion()
198 spin_lock_irqsave(&mes->ring_lock[0], flags); in mes_v11_0_submit_pkt_and_poll_completion()
203 seq = ++ring->fence_drv.sync_seq; in mes_v11_0_submit_pkt_and_poll_completion()
205 seq - ring->fence_drv.num_fences_mask, in mes_v11_0_submit_pkt_and_poll_completion()
211 api_status->api_completion_fence_addr = status_gpu_addr; in mes_v11_0_submit_pkt_and_poll_completion()
212 api_status->api_completion_fence_value = 1; in mes_v11_0_submit_pkt_and_poll_completion()
221 ring->fence_drv.gpu_addr; in mes_v11_0_submit_pkt_and_poll_completion()
228 spin_unlock_irqrestore(&mes->ring_lock[0], flags); in mes_v11_0_submit_pkt_and_poll_completion()
234 dev_dbg(adev->dev, "MES msg=%s (%s) was emitted\n", op_str, in mes_v11_0_submit_pkt_and_poll_completion()
237 dev_dbg(adev->dev, "MES msg=%s was emitted\n", op_str); in mes_v11_0_submit_pkt_and_poll_completion()
239 dev_dbg(adev->dev, "MES msg=%d was emitted\n", in mes_v11_0_submit_pkt_and_poll_completion()
240 x_pkt->header.opcode); in mes_v11_0_submit_pkt_and_poll_completion()
246 dev_err(adev->dev, "MES failed to respond to msg=%s (%s)\n", in mes_v11_0_submit_pkt_and_poll_completion()
249 dev_err(adev->dev, "MES failed to respond to msg=%s\n", in mes_v11_0_submit_pkt_and_poll_completion()
252 dev_err(adev->dev, "MES failed to respond to msg=%d\n", in mes_v11_0_submit_pkt_and_poll_completion()
253 x_pkt->header.opcode); in mes_v11_0_submit_pkt_and_poll_completion()
258 r = -ETIMEDOUT; in mes_v11_0_submit_pkt_and_poll_completion()
266 dev_err(adev->dev, "MES ring buffer is full.\n"); in mes_v11_0_submit_pkt_and_poll_completion()
270 spin_unlock_irqrestore(&mes->ring_lock[0], flags); in mes_v11_0_submit_pkt_and_poll_completion()
287 return -1; in convert_to_mes_queue_type()
293 struct amdgpu_device *adev = mes->adev; in mes_v11_0_add_hw_queue()
295 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; in mes_v11_0_add_hw_queue()
296 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl; in mes_v11_0_add_hw_queue()
304 mes_add_queue_pkt.process_id = input->process_id; in mes_v11_0_add_hw_queue()
305 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr; in mes_v11_0_add_hw_queue()
306 mes_add_queue_pkt.process_va_start = input->process_va_start; in mes_v11_0_add_hw_queue()
307 mes_add_queue_pkt.process_va_end = input->process_va_end; in mes_v11_0_add_hw_queue()
308 mes_add_queue_pkt.process_quantum = input->process_quantum; in mes_v11_0_add_hw_queue()
309 mes_add_queue_pkt.process_context_addr = input->process_context_addr; in mes_v11_0_add_hw_queue()
310 mes_add_queue_pkt.gang_quantum = input->gang_quantum; in mes_v11_0_add_hw_queue()
311 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_add_hw_queue()
313 input->inprocess_gang_priority; in mes_v11_0_add_hw_queue()
315 input->gang_global_priority_level; in mes_v11_0_add_hw_queue()
316 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_add_hw_queue()
317 mes_add_queue_pkt.mqd_addr = input->mqd_addr; in mes_v11_0_add_hw_queue()
319 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> in mes_v11_0_add_hw_queue()
321 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr; in mes_v11_0_add_hw_queue()
323 mes_add_queue_pkt.wptr_addr = input->wptr_addr; in mes_v11_0_add_hw_queue()
326 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_add_hw_queue()
327 mes_add_queue_pkt.paging = input->paging; in mes_v11_0_add_hw_queue()
329 mes_add_queue_pkt.gws_base = input->gws_base; in mes_v11_0_add_hw_queue()
330 mes_add_queue_pkt.gws_size = input->gws_size; in mes_v11_0_add_hw_queue()
331 mes_add_queue_pkt.trap_handler_addr = input->tba_addr; in mes_v11_0_add_hw_queue()
332 mes_add_queue_pkt.tma_addr = input->tma_addr; in mes_v11_0_add_hw_queue()
333 mes_add_queue_pkt.trap_en = input->trap_en; in mes_v11_0_add_hw_queue()
334 mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear; in mes_v11_0_add_hw_queue()
335 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process; in mes_v11_0_add_hw_queue()
337 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */ in mes_v11_0_add_hw_queue()
338 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue; in mes_v11_0_add_hw_queue()
339 mes_add_queue_pkt.gds_size = input->queue_size; in mes_v11_0_add_hw_queue()
341 mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled; in mes_v11_0_add_hw_queue()
359 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_remove_hw_queue()
360 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_remove_hw_queue()
371 struct amdgpu_device *adev = mes->adev; in mes_v11_0_reset_queue_mmio()
378 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n", in mes_v11_0_reset_queue_mmio()
381 mutex_lock(&adev->gfx.reset_sem_mutex); in mes_v11_0_reset_queue_mmio()
393 mutex_unlock(&adev->gfx.reset_sem_mutex); in mes_v11_0_reset_queue_mmio()
395 mutex_lock(&adev->srbm_mutex); in mes_v11_0_reset_queue_mmio()
398 for (i = 0; i < adev->usec_timeout; i++) { in mes_v11_0_reset_queue_mmio()
403 if (i >= adev->usec_timeout) { in mes_v11_0_reset_queue_mmio()
404 dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n"); in mes_v11_0_reset_queue_mmio()
405 r = -ETIMEDOUT; in mes_v11_0_reset_queue_mmio()
409 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_reset_queue_mmio()
411 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n", in mes_v11_0_reset_queue_mmio()
413 mutex_lock(&adev->srbm_mutex); in mes_v11_0_reset_queue_mmio()
419 for (i = 0; i < adev->usec_timeout; i++) { in mes_v11_0_reset_queue_mmio()
424 if (i >= adev->usec_timeout) { in mes_v11_0_reset_queue_mmio()
425 dev_err(adev->dev, "failed to wait on hqd deactivate\n"); in mes_v11_0_reset_queue_mmio()
426 r = -ETIMEDOUT; in mes_v11_0_reset_queue_mmio()
429 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_reset_queue_mmio()
431 dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n", in mes_v11_0_reset_queue_mmio()
446 for (i = 0; i < adev->usec_timeout; i++) { in mes_v11_0_reset_queue_mmio()
451 if (i >= adev->usec_timeout) { in mes_v11_0_reset_queue_mmio()
452 dev_err(adev->dev, "failed to wait on sdma queue reset done\n"); in mes_v11_0_reset_queue_mmio()
453 r = -ETIMEDOUT; in mes_v11_0_reset_queue_mmio()
464 if (input->use_mmio) in mes_v11_0_reset_hw_queue()
465 return mes_v11_0_reset_queue_mmio(mes, input->queue_type, in mes_v11_0_reset_hw_queue()
466 input->me_id, input->pipe_id, in mes_v11_0_reset_hw_queue()
467 input->queue_id, input->vmid); in mes_v11_0_reset_hw_queue()
477 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_reset_hw_queue()
478 mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_reset_hw_queue()
497 mes_add_queue_pkt.pipe_id = input->pipe_id; in mes_v11_0_map_legacy_queue()
498 mes_add_queue_pkt.queue_id = input->queue_id; in mes_v11_0_map_legacy_queue()
499 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_map_legacy_queue()
500 mes_add_queue_pkt.mqd_addr = input->mqd_addr; in mes_v11_0_map_legacy_queue()
501 mes_add_queue_pkt.wptr_addr = input->wptr_addr; in mes_v11_0_map_legacy_queue()
503 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_map_legacy_queue()
522 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_unmap_legacy_queue()
525 mes_remove_queue_pkt.pipe_id = input->pipe_id; in mes_v11_0_unmap_legacy_queue()
526 mes_remove_queue_pkt.queue_id = input->queue_id; in mes_v11_0_unmap_legacy_queue()
528 if (input->action == PREEMPT_QUEUES_NO_UNMAP) { in mes_v11_0_unmap_legacy_queue()
530 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr; in mes_v11_0_unmap_legacy_queue()
532 lower_32_bits(input->trail_fence_data); in mes_v11_0_unmap_legacy_queue()
536 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_unmap_legacy_queue()
555 mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; in mes_v11_0_suspend_gang()
556 mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_suspend_gang()
557 mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; in mes_v11_0_suspend_gang()
558 mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; in mes_v11_0_suspend_gang()
576 mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; in mes_v11_0_resume_gang()
577 mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_resume_gang()
610 switch (input->op) { in mes_v11_0_misc_op()
613 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; in mes_v11_0_misc_op()
614 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; in mes_v11_0_misc_op()
618 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset; in mes_v11_0_misc_op()
619 misc_pkt.write_reg.reg_value = input->write_reg.reg_value; in mes_v11_0_misc_op()
624 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; in mes_v11_0_misc_op()
625 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; in mes_v11_0_misc_op()
626 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; in mes_v11_0_misc_op()
632 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; in mes_v11_0_misc_op()
633 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; in mes_v11_0_misc_op()
634 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; in mes_v11_0_misc_op()
635 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1; in mes_v11_0_misc_op()
640 input->set_shader_debugger.process_context_addr; in mes_v11_0_misc_op()
642 input->set_shader_debugger.flags.u32all; in mes_v11_0_misc_op()
644 input->set_shader_debugger.spi_gdbg_per_vmid_cntl; in mes_v11_0_misc_op()
646 input->set_shader_debugger.tcp_watch_cntl, in mes_v11_0_misc_op()
648 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en; in mes_v11_0_misc_op()
651 if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x63) { in mes_v11_0_misc_op()
652 …dev_err(mes->adev->dev, "MES FW versoin must be larger than 0x63 to support limit single process f… in mes_v11_0_misc_op()
653 return -EINVAL; in mes_v11_0_misc_op()
659 input->change_config.option.limit_single_process; in mes_v11_0_misc_op()
663 DRM_ERROR("unsupported misc op (%d) \n", input->op); in mes_v11_0_misc_op()
664 return -EINVAL; in mes_v11_0_misc_op()
675 struct amdgpu_device *adev = mes->adev; in mes_v11_0_set_hw_resources()
684 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub; in mes_v11_0_set_hw_resources()
685 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub; in mes_v11_0_set_hw_resources()
686 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size; in mes_v11_0_set_hw_resources()
688 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr[0]; in mes_v11_0_set_hw_resources()
690 mes->query_status_fence_gpu_addr[0]; in mes_v11_0_set_hw_resources()
694 mes->compute_hqd_mask[i]; in mes_v11_0_set_hw_resources()
697 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i]; in mes_v11_0_set_hw_resources()
700 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i]; in mes_v11_0_set_hw_resources()
704 mes->aggregated_doorbells[i]; in mes_v11_0_set_hw_resources()
707 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i]; in mes_v11_0_set_hw_resources()
709 adev->reg_offset[MMHUB_HWIP][0][i]; in mes_v11_0_set_hw_resources()
711 adev->reg_offset[OSSSYS_HWIP][0][i]; in mes_v11_0_set_hw_resources()
723 mes->event_log_gpu_addr; in mes_v11_0_set_hw_resources()
744 mes_set_hw_res_pkt.cleaner_shader_fence_mc_addr = mes->resource_1_gpu_addr[0]; in mes_v11_0_set_hw_resources_1()
745 if (amdgpu_sriov_is_mes_info_enable(mes->adev)) { in mes_v11_0_set_hw_resources_1()
747 mes->resource_1_gpu_addr[0] + AMDGPU_GPU_PAGE_SIZE; in mes_v11_0_set_hw_resources_1()
761 if (input->use_mmio) in mes_v11_0_reset_legacy_queue()
762 return mes_v11_0_reset_queue_mmio(mes, input->queue_type, in mes_v11_0_reset_legacy_queue()
763 input->me_id, input->pipe_id, in mes_v11_0_reset_legacy_queue()
764 input->queue_id, input->vmid); in mes_v11_0_reset_legacy_queue()
773 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_reset_legacy_queue()
777 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id; in mes_v11_0_reset_legacy_queue()
778 mes_reset_queue_pkt.queue_id_lp = input->queue_id; in mes_v11_0_reset_legacy_queue()
779 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr; in mes_v11_0_reset_legacy_queue()
780 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset; in mes_v11_0_reset_legacy_queue()
781 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr; in mes_v11_0_reset_legacy_queue()
782 mes_reset_queue_pkt.vmid_id_lp = input->vmid; in mes_v11_0_reset_legacy_queue()
785 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_reset_legacy_queue()
814 adev->mes.fw[pipe]->data; in mes_v11_0_allocate_ucode_buffer()
816 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + in mes_v11_0_allocate_ucode_buffer()
817 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); in mes_v11_0_allocate_ucode_buffer()
818 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); in mes_v11_0_allocate_ucode_buffer()
824 &adev->mes.ucode_fw_obj[pipe], in mes_v11_0_allocate_ucode_buffer()
825 &adev->mes.ucode_fw_gpu_addr[pipe], in mes_v11_0_allocate_ucode_buffer()
826 (void **)&adev->mes.ucode_fw_ptr[pipe]); in mes_v11_0_allocate_ucode_buffer()
828 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r); in mes_v11_0_allocate_ucode_buffer()
832 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); in mes_v11_0_allocate_ucode_buffer()
834 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]); in mes_v11_0_allocate_ucode_buffer()
835 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]); in mes_v11_0_allocate_ucode_buffer()
849 adev->mes.fw[pipe]->data; in mes_v11_0_allocate_ucode_data_buffer()
851 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + in mes_v11_0_allocate_ucode_data_buffer()
852 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); in mes_v11_0_allocate_ucode_data_buffer()
853 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); in mes_v11_0_allocate_ucode_data_buffer()
856 dev_err(adev->dev, "PIPE%d ucode data fw size (%d) is greater than dram size (%d)\n", in mes_v11_0_allocate_ucode_data_buffer()
858 return -EINVAL; in mes_v11_0_allocate_ucode_data_buffer()
865 &adev->mes.data_fw_obj[pipe], in mes_v11_0_allocate_ucode_data_buffer()
866 &adev->mes.data_fw_gpu_addr[pipe], in mes_v11_0_allocate_ucode_data_buffer()
867 (void **)&adev->mes.data_fw_ptr[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
869 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r); in mes_v11_0_allocate_ucode_data_buffer()
873 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size); in mes_v11_0_allocate_ucode_data_buffer()
875 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
876 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
884 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe], in mes_v11_0_free_ucode_buffers()
885 &adev->mes.data_fw_gpu_addr[pipe], in mes_v11_0_free_ucode_buffers()
886 (void **)&adev->mes.data_fw_ptr[pipe]); in mes_v11_0_free_ucode_buffers()
888 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe], in mes_v11_0_free_ucode_buffers()
889 &adev->mes.ucode_fw_gpu_addr[pipe], in mes_v11_0_free_ucode_buffers()
890 (void **)&adev->mes.ucode_fw_ptr[pipe]); in mes_v11_0_free_ucode_buffers()
898 if (adev->mes.sched_version && adev->mes.kiq_version) in mes_v11_0_get_fw_version()
902 mutex_lock(&adev->srbm_mutex); in mes_v11_0_get_fw_version()
908 adev->mes.sched_version = in mes_v11_0_get_fw_version()
910 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) in mes_v11_0_get_fw_version()
911 adev->mes.kiq_version = in mes_v11_0_get_fw_version()
916 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_get_fw_version()
927 lower_32_bits(adev->mes.event_log_gpu_addr + AMDGPU_MES_LOG_BUFFER_SIZE)); in mes_v11_0_enable()
929 upper_32_bits(adev->mes.event_log_gpu_addr + AMDGPU_MES_LOG_BUFFER_SIZE)); in mes_v11_0_enable()
930 dev_info(adev->dev, "Setup CP MES MSCRATCH address : 0x%x. 0x%x\n", in mes_v11_0_enable()
938 MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
941 mutex_lock(&adev->srbm_mutex); in mes_v11_0_enable()
943 if (!adev->enable_mes_kiq && in mes_v11_0_enable()
949 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; in mes_v11_0_enable()
956 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_enable()
961 adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
976 adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
992 if (!adev->mes.fw[pipe]) in mes_v11_0_load_microcode()
993 return -EINVAL; in mes_v11_0_load_microcode()
1005 mutex_lock(&adev->srbm_mutex); in mes_v11_0_load_microcode()
1012 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; in mes_v11_0_load_microcode()
1020 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
1022 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
1024 /* set ucode instruction cache boundary to 2M-1 */ in mes_v11_0_load_microcode()
1029 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
1031 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
1033 /* Set 0x7FFFF (512K-1) to CP_MES_MDBOUND_LO */ in mes_v11_0_load_microcode()
1050 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_load_microcode()
1059 u32 *eop; in mes_v11_0_allocate_eop_buf() local
1063 &adev->mes.eop_gpu_obj[pipe], in mes_v11_0_allocate_eop_buf()
1064 &adev->mes.eop_gpu_addr[pipe], in mes_v11_0_allocate_eop_buf()
1065 (void **)&eop); in mes_v11_0_allocate_eop_buf()
1067 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r); in mes_v11_0_allocate_eop_buf()
1071 memset(eop, 0, in mes_v11_0_allocate_eop_buf()
1072 adev->mes.eop_gpu_obj[pipe]->tbo.base.size); in mes_v11_0_allocate_eop_buf()
1074 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]); in mes_v11_0_allocate_eop_buf()
1075 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]); in mes_v11_0_allocate_eop_buf()
1082 struct v11_compute_mqd *mqd = ring->mqd_ptr; in mes_v11_0_mqd_init()
1088 mqd->header = 0xC0310800; in mes_v11_0_mqd_init()
1089 mqd->compute_pipelinestat_enable = 0x00000001; in mes_v11_0_mqd_init()
1090 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in mes_v11_0_mqd_init()
1091 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in mes_v11_0_mqd_init()
1092 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in mes_v11_0_mqd_init()
1093 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in mes_v11_0_mqd_init()
1094 mqd->compute_misc_reserved = 0x00000007; in mes_v11_0_mqd_init()
1096 eop_base_addr = ring->eop_gpu_addr >> 8; in mes_v11_0_mqd_init()
1098 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ in mes_v11_0_mqd_init()
1101 (order_base_2(MES_EOP_SIZE / 4) - 1)); in mes_v11_0_mqd_init()
1103 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr); in mes_v11_0_mqd_init()
1104 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); in mes_v11_0_mqd_init()
1105 mqd->cp_hqd_eop_control = tmp; in mes_v11_0_mqd_init()
1107 /* disable the queue if it's active */ in mes_v11_0_mqd_init()
1108 ring->wptr = 0; in mes_v11_0_mqd_init()
1109 mqd->cp_hqd_pq_rptr = 0; in mes_v11_0_mqd_init()
1110 mqd->cp_hqd_pq_wptr_lo = 0; in mes_v11_0_mqd_init()
1111 mqd->cp_hqd_pq_wptr_hi = 0; in mes_v11_0_mqd_init()
1114 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; in mes_v11_0_mqd_init()
1115 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); in mes_v11_0_mqd_init()
1120 mqd->cp_mqd_control = tmp; in mes_v11_0_mqd_init()
1123 hqd_gpu_addr = ring->gpu_addr >> 8; in mes_v11_0_mqd_init()
1124 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr); in mes_v11_0_mqd_init()
1125 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); in mes_v11_0_mqd_init()
1128 wb_gpu_addr = ring->rptr_gpu_addr; in mes_v11_0_mqd_init()
1129 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in mes_v11_0_mqd_init()
1130 mqd->cp_hqd_pq_rptr_report_addr_hi = in mes_v11_0_mqd_init()
1134 wb_gpu_addr = ring->wptr_gpu_addr; in mes_v11_0_mqd_init()
1135 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8; in mes_v11_0_mqd_init()
1136 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in mes_v11_0_mqd_init()
1141 (order_base_2(ring->ring_size / 4) - 1)); in mes_v11_0_mqd_init()
1143 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); in mes_v11_0_mqd_init()
1149 mqd->cp_hqd_pq_control = tmp; in mes_v11_0_mqd_init()
1153 if (ring->use_doorbell) { in mes_v11_0_mqd_init()
1155 DOORBELL_OFFSET, ring->doorbell_index); in mes_v11_0_mqd_init()
1165 mqd->cp_hqd_pq_doorbell_control = tmp; in mes_v11_0_mqd_init()
1167 mqd->cp_hqd_vmid = 0; in mes_v11_0_mqd_init()
1169 mqd->cp_hqd_active = 1; in mes_v11_0_mqd_init()
1174 mqd->cp_hqd_persistent_state = tmp; in mes_v11_0_mqd_init()
1176 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT; in mes_v11_0_mqd_init()
1177 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT; in mes_v11_0_mqd_init()
1178 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT; in mes_v11_0_mqd_init()
1180 amdgpu_device_flush_hdp(ring->adev, NULL); in mes_v11_0_mqd_init()
1186 struct v11_compute_mqd *mqd = ring->mqd_ptr; in mes_v11_0_queue_init_register()
1187 struct amdgpu_device *adev = ring->adev; in mes_v11_0_queue_init_register()
1190 mutex_lock(&adev->srbm_mutex); in mes_v11_0_queue_init_register()
1191 soc21_grbm_select(adev, 3, ring->pipe, 0, 0); in mes_v11_0_queue_init_register()
1205 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo); in mes_v11_0_queue_init_register()
1206 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi); in mes_v11_0_queue_init_register()
1214 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo); in mes_v11_0_queue_init_register()
1215 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi); in mes_v11_0_queue_init_register()
1219 mqd->cp_hqd_pq_rptr_report_addr_lo); in mes_v11_0_queue_init_register()
1221 mqd->cp_hqd_pq_rptr_report_addr_hi); in mes_v11_0_queue_init_register()
1224 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control); in mes_v11_0_queue_init_register()
1228 mqd->cp_hqd_pq_wptr_poll_addr_lo); in mes_v11_0_queue_init_register()
1230 mqd->cp_hqd_pq_wptr_poll_addr_hi); in mes_v11_0_queue_init_register()
1234 mqd->cp_hqd_pq_doorbell_control); in mes_v11_0_queue_init_register()
1237 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state); in mes_v11_0_queue_init_register()
1240 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active); in mes_v11_0_queue_init_register()
1243 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_queue_init_register()
1248 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in mes_v11_0_kiq_enable_queue()
1249 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; in mes_v11_0_kiq_enable_queue()
1252 if (!kiq->pmf || !kiq->pmf->kiq_map_queues) in mes_v11_0_kiq_enable_queue()
1253 return -EINVAL; in mes_v11_0_kiq_enable_queue()
1255 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size); in mes_v11_0_kiq_enable_queue()
1261 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]); in mes_v11_0_kiq_enable_queue()
1273 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_queue_init()
1275 ring = &adev->mes.ring[0]; in mes_v11_0_queue_init()
1280 (amdgpu_in_reset(adev) || adev->in_suspend)) { in mes_v11_0_queue_init()
1281 *(ring->wptr_cpu_addr) = 0; in mes_v11_0_queue_init()
1282 *(ring->rptr_cpu_addr) = 0; in mes_v11_0_queue_init()
1305 ring = &adev->mes.ring[0]; in mes_v11_0_ring_init()
1307 ring->funcs = &mes_v11_0_ring_funcs; in mes_v11_0_ring_init()
1309 ring->me = 3; in mes_v11_0_ring_init()
1310 ring->pipe = 0; in mes_v11_0_ring_init()
1311 ring->queue = 0; in mes_v11_0_ring_init()
1313 ring->ring_obj = NULL; in mes_v11_0_ring_init()
1314 ring->use_doorbell = true; in mes_v11_0_ring_init()
1315 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1; in mes_v11_0_ring_init()
1316 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE]; in mes_v11_0_ring_init()
1317 ring->no_scheduler = true; in mes_v11_0_ring_init()
1318 sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue); in mes_v11_0_ring_init()
1328 spin_lock_init(&adev->gfx.kiq[0].ring_lock); in mes_v11_0_kiq_ring_init()
1330 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_kiq_ring_init()
1332 ring->me = 3; in mes_v11_0_kiq_ring_init()
1333 ring->pipe = 1; in mes_v11_0_kiq_ring_init()
1334 ring->queue = 0; in mes_v11_0_kiq_ring_init()
1336 ring->adev = NULL; in mes_v11_0_kiq_ring_init()
1337 ring->ring_obj = NULL; in mes_v11_0_kiq_ring_init()
1338 ring->use_doorbell = true; in mes_v11_0_kiq_ring_init()
1339 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1; in mes_v11_0_kiq_ring_init()
1340 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE]; in mes_v11_0_kiq_ring_init()
1341 ring->no_scheduler = true; in mes_v11_0_kiq_ring_init()
1342 sprintf(ring->name, "mes_kiq_%d.%d.%d", in mes_v11_0_kiq_ring_init()
1343 ring->me, ring->pipe, ring->queue); in mes_v11_0_kiq_ring_init()
1356 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_mqd_sw_init()
1358 ring = &adev->mes.ring[0]; in mes_v11_0_mqd_sw_init()
1362 if (ring->mqd_obj) in mes_v11_0_mqd_sw_init()
1367 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, in mes_v11_0_mqd_sw_init()
1368 &ring->mqd_gpu_addr, &ring->mqd_ptr); in mes_v11_0_mqd_sw_init()
1370 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); in mes_v11_0_mqd_sw_init()
1374 memset(ring->mqd_ptr, 0, mqd_size); in mes_v11_0_mqd_sw_init()
1377 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL); in mes_v11_0_mqd_sw_init()
1378 if (!adev->mes.mqd_backup[pipe]) { in mes_v11_0_mqd_sw_init()
1379 dev_warn(adev->dev, in mes_v11_0_mqd_sw_init()
1381 ring->name); in mes_v11_0_mqd_sw_init()
1382 return -ENOMEM; in mes_v11_0_mqd_sw_init()
1390 struct amdgpu_device *adev = ip_block->adev; in mes_v11_0_sw_init()
1393 adev->mes.funcs = &mes_v11_0_funcs; in mes_v11_0_sw_init()
1394 adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init; in mes_v11_0_sw_init()
1395 adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini; in mes_v11_0_sw_init()
1397 adev->mes.event_log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE; in mes_v11_0_sw_init()
1404 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) in mes_v11_0_sw_init()
1416 if (adev->enable_mes_kiq) { in mes_v11_0_sw_init()
1435 &adev->mes.resource_1[0], in mes_v11_0_sw_init()
1436 &adev->mes.resource_1_gpu_addr[0], in mes_v11_0_sw_init()
1437 &adev->mes.resource_1_addr[0]); in mes_v11_0_sw_init()
1439 dev_err(adev->dev, "(%d) failed to create mes resource_1 bo\n", r); in mes_v11_0_sw_init()
1448 struct amdgpu_device *adev = ip_block->adev; in mes_v11_0_sw_fini()
1451 amdgpu_bo_free_kernel(&adev->mes.resource_1[0], &adev->mes.resource_1_gpu_addr[0], in mes_v11_0_sw_fini()
1452 &adev->mes.resource_1_addr[0]); in mes_v11_0_sw_fini()
1455 kfree(adev->mes.mqd_backup[pipe]); in mes_v11_0_sw_fini()
1457 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe], in mes_v11_0_sw_fini()
1458 &adev->mes.eop_gpu_addr[pipe], in mes_v11_0_sw_fini()
1460 amdgpu_ucode_release(&adev->mes.fw[pipe]); in mes_v11_0_sw_fini()
1463 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj, in mes_v11_0_sw_fini()
1464 &adev->gfx.kiq[0].ring.mqd_gpu_addr, in mes_v11_0_sw_fini()
1465 &adev->gfx.kiq[0].ring.mqd_ptr); in mes_v11_0_sw_fini()
1467 amdgpu_bo_free_kernel(&adev->mes.ring[0].mqd_obj, in mes_v11_0_sw_fini()
1468 &adev->mes.ring[0].mqd_gpu_addr, in mes_v11_0_sw_fini()
1469 &adev->mes.ring[0].mqd_ptr); in mes_v11_0_sw_fini()
1471 amdgpu_ring_fini(&adev->gfx.kiq[0].ring); in mes_v11_0_sw_fini()
1472 amdgpu_ring_fini(&adev->mes.ring[0]); in mes_v11_0_sw_fini()
1474 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_sw_fini()
1487 struct amdgpu_device *adev = ring->adev; in mes_v11_0_kiq_dequeue()
1489 mutex_lock(&adev->srbm_mutex); in mes_v11_0_kiq_dequeue()
1490 soc21_grbm_select(adev, 3, ring->pipe, 0, 0); in mes_v11_0_kiq_dequeue()
1492 /* disable the queue if it's active */ in mes_v11_0_kiq_dequeue()
1495 for (i = 0; i < adev->usec_timeout; i++) { in mes_v11_0_kiq_dequeue()
1515 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_kiq_dequeue()
1521 struct amdgpu_device *adev = ring->adev; in mes_v11_0_kiq_setting()
1526 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); in mes_v11_0_kiq_setting()
1545 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_kiq_hw_init()
1565 mes_v11_0_kiq_setting(&adev->gfx.kiq[0].ring); in mes_v11_0_kiq_hw_init()
1569 dev_err(adev->dev, "Failed to get MES handle\n"); in mes_v11_0_kiq_hw_init()
1570 return -EINVAL; in mes_v11_0_kiq_hw_init()
1577 if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x47) in mes_v11_0_kiq_hw_init()
1578 adev->mes.enable_legacy_queue_map = true; in mes_v11_0_kiq_hw_init()
1580 adev->mes.enable_legacy_queue_map = false; in mes_v11_0_kiq_hw_init()
1582 if (adev->mes.enable_legacy_queue_map) { in mes_v11_0_kiq_hw_init()
1597 if (adev->mes.ring[0].sched.ready) { in mes_v11_0_kiq_hw_fini()
1598 mes_v11_0_kiq_dequeue(&adev->mes.ring[0]); in mes_v11_0_kiq_hw_fini()
1599 adev->mes.ring[0].sched.ready = false; in mes_v11_0_kiq_hw_fini()
1603 mes_v11_0_kiq_dequeue(&adev->gfx.kiq[0].ring); in mes_v11_0_kiq_hw_fini()
1615 struct amdgpu_device *adev = ip_block->adev; in mes_v11_0_hw_init()
1617 if (adev->mes.ring[0].sched.ready) in mes_v11_0_hw_init()
1620 if (!adev->enable_mes_kiq) { in mes_v11_0_hw_init()
1621 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_hw_init()
1637 r = mes_v11_0_set_hw_resources(&adev->mes); in mes_v11_0_hw_init()
1641 r = mes_v11_0_set_hw_resources_1(&adev->mes); in mes_v11_0_hw_init()
1647 r = mes_v11_0_query_sched_status(&adev->mes); in mes_v11_0_hw_init()
1659 * Disable KIQ ring usage from the driver once MES is enabled. in mes_v11_0_hw_init()
1663 adev->gfx.kiq[0].ring.sched.ready = false; in mes_v11_0_hw_init()
1664 adev->mes.ring[0].sched.ready = true; in mes_v11_0_hw_init()
1690 struct amdgpu_device *adev = ip_block->adev; in mes_v11_0_early_init()
1694 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) in mes_v11_0_early_init()
1706 struct amdgpu_device *adev = ip_block->adev; in mes_v11_0_late_init()
1709 if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend && in mes_v11_0_late_init()