Lines Matching +full:disable +full:- +full:eop
62 struct amdgpu_device *adev = ring->adev; in mes_v11_0_ring_set_wptr()
64 if (ring->use_doorbell) { in mes_v11_0_ring_set_wptr()
65 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, in mes_v11_0_ring_set_wptr()
66 ring->wptr); in mes_v11_0_ring_set_wptr()
67 WDOORBELL64(ring->doorbell_index, ring->wptr); in mes_v11_0_ring_set_wptr()
75 return *ring->rptr_cpu_addr; in mes_v11_0_ring_get_rptr()
82 if (ring->use_doorbell) in mes_v11_0_ring_get_wptr()
83 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); in mes_v11_0_ring_get_wptr()
108 struct amdgpu_device *adev = mes->adev; in mes_v11_0_submit_pkt_and_poll_completion()
109 struct amdgpu_ring *ring = &mes->ring; in mes_v11_0_submit_pkt_and_poll_completion()
111 signed long timeout = adev->usec_timeout; in mes_v11_0_submit_pkt_and_poll_completion()
121 spin_lock_irqsave(&mes->ring_lock, flags); in mes_v11_0_submit_pkt_and_poll_completion()
123 spin_unlock_irqrestore(&mes->ring_lock, flags); in mes_v11_0_submit_pkt_and_poll_completion()
124 return -ENOMEM; in mes_v11_0_submit_pkt_and_poll_completion()
128 api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr; in mes_v11_0_submit_pkt_and_poll_completion()
129 api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq; in mes_v11_0_submit_pkt_and_poll_completion()
133 spin_unlock_irqrestore(&mes->ring_lock, flags); in mes_v11_0_submit_pkt_and_poll_completion()
135 DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode); in mes_v11_0_submit_pkt_and_poll_completion()
137 r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, in mes_v11_0_submit_pkt_and_poll_completion()
141 x_pkt->header.opcode); in mes_v11_0_submit_pkt_and_poll_completion()
146 return -ETIMEDOUT; in mes_v11_0_submit_pkt_and_poll_completion()
162 return -1; in convert_to_mes_queue_type()
168 struct amdgpu_device *adev = mes->adev; in mes_v11_0_add_hw_queue()
170 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; in mes_v11_0_add_hw_queue()
171 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl; in mes_v11_0_add_hw_queue()
179 mes_add_queue_pkt.process_id = input->process_id; in mes_v11_0_add_hw_queue()
180 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr; in mes_v11_0_add_hw_queue()
181 mes_add_queue_pkt.process_va_start = input->process_va_start; in mes_v11_0_add_hw_queue()
182 mes_add_queue_pkt.process_va_end = input->process_va_end; in mes_v11_0_add_hw_queue()
183 mes_add_queue_pkt.process_quantum = input->process_quantum; in mes_v11_0_add_hw_queue()
184 mes_add_queue_pkt.process_context_addr = input->process_context_addr; in mes_v11_0_add_hw_queue()
185 mes_add_queue_pkt.gang_quantum = input->gang_quantum; in mes_v11_0_add_hw_queue()
186 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_add_hw_queue()
188 input->inprocess_gang_priority; in mes_v11_0_add_hw_queue()
190 input->gang_global_priority_level; in mes_v11_0_add_hw_queue()
191 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_add_hw_queue()
192 mes_add_queue_pkt.mqd_addr = input->mqd_addr; in mes_v11_0_add_hw_queue()
194 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> in mes_v11_0_add_hw_queue()
196 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr; in mes_v11_0_add_hw_queue()
198 mes_add_queue_pkt.wptr_addr = input->wptr_addr; in mes_v11_0_add_hw_queue()
201 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_add_hw_queue()
202 mes_add_queue_pkt.paging = input->paging; in mes_v11_0_add_hw_queue()
204 mes_add_queue_pkt.gws_base = input->gws_base; in mes_v11_0_add_hw_queue()
205 mes_add_queue_pkt.gws_size = input->gws_size; in mes_v11_0_add_hw_queue()
206 mes_add_queue_pkt.trap_handler_addr = input->tba_addr; in mes_v11_0_add_hw_queue()
207 mes_add_queue_pkt.tma_addr = input->tma_addr; in mes_v11_0_add_hw_queue()
208 mes_add_queue_pkt.trap_en = input->trap_en; in mes_v11_0_add_hw_queue()
209 mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear; in mes_v11_0_add_hw_queue()
210 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process; in mes_v11_0_add_hw_queue()
212 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */ in mes_v11_0_add_hw_queue()
213 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue; in mes_v11_0_add_hw_queue()
214 mes_add_queue_pkt.gds_size = input->queue_size; in mes_v11_0_add_hw_queue()
216 mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled; in mes_v11_0_add_hw_queue()
234 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_remove_hw_queue()
235 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; in mes_v11_0_remove_hw_queue()
253 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; in mes_v11_0_unmap_legacy_queue()
256 mes_remove_queue_pkt.pipe_id = input->pipe_id; in mes_v11_0_unmap_legacy_queue()
257 mes_remove_queue_pkt.queue_id = input->queue_id; in mes_v11_0_unmap_legacy_queue()
259 if (input->action == PREEMPT_QUEUES_NO_UNMAP) { in mes_v11_0_unmap_legacy_queue()
261 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr; in mes_v11_0_unmap_legacy_queue()
263 lower_32_bits(input->trail_fence_data); in mes_v11_0_unmap_legacy_queue()
267 convert_to_mes_queue_type(input->queue_type); in mes_v11_0_unmap_legacy_queue()
313 switch (input->op) { in mes_v11_0_misc_op()
316 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; in mes_v11_0_misc_op()
317 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; in mes_v11_0_misc_op()
321 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset; in mes_v11_0_misc_op()
322 misc_pkt.write_reg.reg_value = input->write_reg.reg_value; in mes_v11_0_misc_op()
327 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; in mes_v11_0_misc_op()
328 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; in mes_v11_0_misc_op()
329 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; in mes_v11_0_misc_op()
335 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; in mes_v11_0_misc_op()
336 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; in mes_v11_0_misc_op()
337 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; in mes_v11_0_misc_op()
338 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1; in mes_v11_0_misc_op()
343 input->set_shader_debugger.process_context_addr; in mes_v11_0_misc_op()
345 input->set_shader_debugger.flags.u32all; in mes_v11_0_misc_op()
347 input->set_shader_debugger.spi_gdbg_per_vmid_cntl; in mes_v11_0_misc_op()
349 input->set_shader_debugger.tcp_watch_cntl, in mes_v11_0_misc_op()
351 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en; in mes_v11_0_misc_op()
354 DRM_ERROR("unsupported misc op (%d) \n", input->op); in mes_v11_0_misc_op()
355 return -EINVAL; in mes_v11_0_misc_op()
366 struct amdgpu_device *adev = mes->adev; in mes_v11_0_set_hw_resources()
375 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub; in mes_v11_0_set_hw_resources()
376 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub; in mes_v11_0_set_hw_resources()
377 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size; in mes_v11_0_set_hw_resources()
379 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr; in mes_v11_0_set_hw_resources()
381 mes->query_status_fence_gpu_addr; in mes_v11_0_set_hw_resources()
385 mes->compute_hqd_mask[i]; in mes_v11_0_set_hw_resources()
388 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i]; in mes_v11_0_set_hw_resources()
391 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i]; in mes_v11_0_set_hw_resources()
395 mes->aggregated_doorbells[i]; in mes_v11_0_set_hw_resources()
398 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i]; in mes_v11_0_set_hw_resources()
400 adev->reg_offset[MMHUB_HWIP][0][i]; in mes_v11_0_set_hw_resources()
402 adev->reg_offset[OSSSYS_HWIP][0][i]; in mes_v11_0_set_hw_resources()
412 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr; in mes_v11_0_set_hw_resources()
437 adev->mes.fw[pipe]->data; in mes_v11_0_allocate_ucode_buffer()
439 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + in mes_v11_0_allocate_ucode_buffer()
440 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); in mes_v11_0_allocate_ucode_buffer()
441 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); in mes_v11_0_allocate_ucode_buffer()
447 &adev->mes.ucode_fw_obj[pipe], in mes_v11_0_allocate_ucode_buffer()
448 &adev->mes.ucode_fw_gpu_addr[pipe], in mes_v11_0_allocate_ucode_buffer()
449 (void **)&adev->mes.ucode_fw_ptr[pipe]); in mes_v11_0_allocate_ucode_buffer()
451 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r); in mes_v11_0_allocate_ucode_buffer()
455 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); in mes_v11_0_allocate_ucode_buffer()
457 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]); in mes_v11_0_allocate_ucode_buffer()
458 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]); in mes_v11_0_allocate_ucode_buffer()
472 adev->mes.fw[pipe]->data; in mes_v11_0_allocate_ucode_data_buffer()
474 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + in mes_v11_0_allocate_ucode_data_buffer()
475 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); in mes_v11_0_allocate_ucode_data_buffer()
476 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); in mes_v11_0_allocate_ucode_data_buffer()
482 &adev->mes.data_fw_obj[pipe], in mes_v11_0_allocate_ucode_data_buffer()
483 &adev->mes.data_fw_gpu_addr[pipe], in mes_v11_0_allocate_ucode_data_buffer()
484 (void **)&adev->mes.data_fw_ptr[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
486 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r); in mes_v11_0_allocate_ucode_data_buffer()
490 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size); in mes_v11_0_allocate_ucode_data_buffer()
492 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
493 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]); in mes_v11_0_allocate_ucode_data_buffer()
501 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe], in mes_v11_0_free_ucode_buffers()
502 &adev->mes.data_fw_gpu_addr[pipe], in mes_v11_0_free_ucode_buffers()
503 (void **)&adev->mes.data_fw_ptr[pipe]); in mes_v11_0_free_ucode_buffers()
505 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe], in mes_v11_0_free_ucode_buffers()
506 &adev->mes.ucode_fw_gpu_addr[pipe], in mes_v11_0_free_ucode_buffers()
507 (void **)&adev->mes.ucode_fw_ptr[pipe]); in mes_v11_0_free_ucode_buffers()
519 MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
522 mutex_lock(&adev->srbm_mutex); in mes_v11_0_enable()
524 if (!adev->enable_mes_kiq && in mes_v11_0_enable()
530 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; in mes_v11_0_enable()
537 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_enable()
542 adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
557 adev->enable_mes_kiq ? 1 : 0); in mes_v11_0_enable()
573 if (!adev->mes.fw[pipe]) in mes_v11_0_load_microcode()
574 return -EINVAL; in mes_v11_0_load_microcode()
586 mutex_lock(&adev->srbm_mutex); in mes_v11_0_load_microcode()
593 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; in mes_v11_0_load_microcode()
601 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
603 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
605 /* set ucode instruction cache boundary to 2M-1 */ in mes_v11_0_load_microcode()
610 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
612 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe])); in mes_v11_0_load_microcode()
614 /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */ in mes_v11_0_load_microcode()
631 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_load_microcode()
640 u32 *eop; in mes_v11_0_allocate_eop_buf() local
644 &adev->mes.eop_gpu_obj[pipe], in mes_v11_0_allocate_eop_buf()
645 &adev->mes.eop_gpu_addr[pipe], in mes_v11_0_allocate_eop_buf()
646 (void **)&eop); in mes_v11_0_allocate_eop_buf()
648 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r); in mes_v11_0_allocate_eop_buf()
652 memset(eop, 0, in mes_v11_0_allocate_eop_buf()
653 adev->mes.eop_gpu_obj[pipe]->tbo.base.size); in mes_v11_0_allocate_eop_buf()
655 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]); in mes_v11_0_allocate_eop_buf()
656 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]); in mes_v11_0_allocate_eop_buf()
663 struct v11_compute_mqd *mqd = ring->mqd_ptr; in mes_v11_0_mqd_init()
669 mqd->header = 0xC0310800; in mes_v11_0_mqd_init()
670 mqd->compute_pipelinestat_enable = 0x00000001; in mes_v11_0_mqd_init()
671 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in mes_v11_0_mqd_init()
672 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in mes_v11_0_mqd_init()
673 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in mes_v11_0_mqd_init()
674 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in mes_v11_0_mqd_init()
675 mqd->compute_misc_reserved = 0x00000007; in mes_v11_0_mqd_init()
677 eop_base_addr = ring->eop_gpu_addr >> 8; in mes_v11_0_mqd_init()
679 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ in mes_v11_0_mqd_init()
682 (order_base_2(MES_EOP_SIZE / 4) - 1)); in mes_v11_0_mqd_init()
684 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr); in mes_v11_0_mqd_init()
685 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); in mes_v11_0_mqd_init()
686 mqd->cp_hqd_eop_control = tmp; in mes_v11_0_mqd_init()
688 /* disable the queue if it's active */ in mes_v11_0_mqd_init()
689 ring->wptr = 0; in mes_v11_0_mqd_init()
690 mqd->cp_hqd_pq_rptr = 0; in mes_v11_0_mqd_init()
691 mqd->cp_hqd_pq_wptr_lo = 0; in mes_v11_0_mqd_init()
692 mqd->cp_hqd_pq_wptr_hi = 0; in mes_v11_0_mqd_init()
695 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; in mes_v11_0_mqd_init()
696 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); in mes_v11_0_mqd_init()
701 mqd->cp_mqd_control = tmp; in mes_v11_0_mqd_init()
704 hqd_gpu_addr = ring->gpu_addr >> 8; in mes_v11_0_mqd_init()
705 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr); in mes_v11_0_mqd_init()
706 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); in mes_v11_0_mqd_init()
709 wb_gpu_addr = ring->rptr_gpu_addr; in mes_v11_0_mqd_init()
710 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in mes_v11_0_mqd_init()
711 mqd->cp_hqd_pq_rptr_report_addr_hi = in mes_v11_0_mqd_init()
715 wb_gpu_addr = ring->wptr_gpu_addr; in mes_v11_0_mqd_init()
716 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8; in mes_v11_0_mqd_init()
717 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in mes_v11_0_mqd_init()
722 (order_base_2(ring->ring_size / 4) - 1)); in mes_v11_0_mqd_init()
724 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); in mes_v11_0_mqd_init()
730 mqd->cp_hqd_pq_control = tmp; in mes_v11_0_mqd_init()
734 if (ring->use_doorbell) { in mes_v11_0_mqd_init()
736 DOORBELL_OFFSET, ring->doorbell_index); in mes_v11_0_mqd_init()
746 mqd->cp_hqd_pq_doorbell_control = tmp; in mes_v11_0_mqd_init()
748 mqd->cp_hqd_vmid = 0; in mes_v11_0_mqd_init()
750 mqd->cp_hqd_active = 1; in mes_v11_0_mqd_init()
755 mqd->cp_hqd_persistent_state = tmp; in mes_v11_0_mqd_init()
757 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT; in mes_v11_0_mqd_init()
758 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT; in mes_v11_0_mqd_init()
759 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT; in mes_v11_0_mqd_init()
761 amdgpu_device_flush_hdp(ring->adev, NULL); in mes_v11_0_mqd_init()
767 struct v11_compute_mqd *mqd = ring->mqd_ptr; in mes_v11_0_queue_init_register()
768 struct amdgpu_device *adev = ring->adev; in mes_v11_0_queue_init_register()
771 mutex_lock(&adev->srbm_mutex); in mes_v11_0_queue_init_register()
772 soc21_grbm_select(adev, 3, ring->pipe, 0, 0); in mes_v11_0_queue_init_register()
786 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo); in mes_v11_0_queue_init_register()
787 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi); in mes_v11_0_queue_init_register()
795 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo); in mes_v11_0_queue_init_register()
796 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi); in mes_v11_0_queue_init_register()
800 mqd->cp_hqd_pq_rptr_report_addr_lo); in mes_v11_0_queue_init_register()
802 mqd->cp_hqd_pq_rptr_report_addr_hi); in mes_v11_0_queue_init_register()
805 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control); in mes_v11_0_queue_init_register()
809 mqd->cp_hqd_pq_wptr_poll_addr_lo); in mes_v11_0_queue_init_register()
811 mqd->cp_hqd_pq_wptr_poll_addr_hi); in mes_v11_0_queue_init_register()
815 mqd->cp_hqd_pq_doorbell_control); in mes_v11_0_queue_init_register()
818 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state); in mes_v11_0_queue_init_register()
821 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active); in mes_v11_0_queue_init_register()
824 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_queue_init_register()
829 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in mes_v11_0_kiq_enable_queue()
830 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; in mes_v11_0_kiq_enable_queue()
833 if (!kiq->pmf || !kiq->pmf->kiq_map_queues) in mes_v11_0_kiq_enable_queue()
834 return -EINVAL; in mes_v11_0_kiq_enable_queue()
836 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size); in mes_v11_0_kiq_enable_queue()
842 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring); in mes_v11_0_kiq_enable_queue()
854 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_queue_init()
856 ring = &adev->mes.ring; in mes_v11_0_queue_init()
861 (amdgpu_in_reset(adev) || adev->in_suspend)) { in mes_v11_0_queue_init()
862 *(ring->wptr_cpu_addr) = 0; in mes_v11_0_queue_init()
863 *(ring->rptr_cpu_addr) = 0; in mes_v11_0_queue_init()
880 mutex_lock(&adev->srbm_mutex); in mes_v11_0_queue_init()
884 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); in mes_v11_0_queue_init()
885 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) in mes_v11_0_queue_init()
886 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); in mes_v11_0_queue_init()
889 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_queue_init()
898 ring = &adev->mes.ring; in mes_v11_0_ring_init()
900 ring->funcs = &mes_v11_0_ring_funcs; in mes_v11_0_ring_init()
902 ring->me = 3; in mes_v11_0_ring_init()
903 ring->pipe = 0; in mes_v11_0_ring_init()
904 ring->queue = 0; in mes_v11_0_ring_init()
906 ring->ring_obj = NULL; in mes_v11_0_ring_init()
907 ring->use_doorbell = true; in mes_v11_0_ring_init()
908 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1; in mes_v11_0_ring_init()
909 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE]; in mes_v11_0_ring_init()
910 ring->no_scheduler = true; in mes_v11_0_ring_init()
911 sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue); in mes_v11_0_ring_init()
921 spin_lock_init(&adev->gfx.kiq[0].ring_lock); in mes_v11_0_kiq_ring_init()
923 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_kiq_ring_init()
925 ring->me = 3; in mes_v11_0_kiq_ring_init()
926 ring->pipe = 1; in mes_v11_0_kiq_ring_init()
927 ring->queue = 0; in mes_v11_0_kiq_ring_init()
929 ring->adev = NULL; in mes_v11_0_kiq_ring_init()
930 ring->ring_obj = NULL; in mes_v11_0_kiq_ring_init()
931 ring->use_doorbell = true; in mes_v11_0_kiq_ring_init()
932 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1; in mes_v11_0_kiq_ring_init()
933 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE]; in mes_v11_0_kiq_ring_init()
934 ring->no_scheduler = true; in mes_v11_0_kiq_ring_init()
935 sprintf(ring->name, "mes_kiq_%d.%d.%d", in mes_v11_0_kiq_ring_init()
936 ring->me, ring->pipe, ring->queue); in mes_v11_0_kiq_ring_init()
949 ring = &adev->gfx.kiq[0].ring; in mes_v11_0_mqd_sw_init()
951 ring = &adev->mes.ring; in mes_v11_0_mqd_sw_init()
955 if (ring->mqd_obj) in mes_v11_0_mqd_sw_init()
960 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, in mes_v11_0_mqd_sw_init()
961 &ring->mqd_gpu_addr, &ring->mqd_ptr); in mes_v11_0_mqd_sw_init()
963 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); in mes_v11_0_mqd_sw_init()
967 memset(ring->mqd_ptr, 0, mqd_size); in mes_v11_0_mqd_sw_init()
970 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL); in mes_v11_0_mqd_sw_init()
971 if (!adev->mes.mqd_backup[pipe]) { in mes_v11_0_mqd_sw_init()
972 dev_warn(adev->dev, in mes_v11_0_mqd_sw_init()
974 ring->name); in mes_v11_0_mqd_sw_init()
975 return -ENOMEM; in mes_v11_0_mqd_sw_init()
986 adev->mes.funcs = &mes_v11_0_funcs; in mes_v11_0_sw_init()
987 adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init; in mes_v11_0_sw_init()
988 adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini; in mes_v11_0_sw_init()
995 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) in mes_v11_0_sw_init()
1007 if (adev->enable_mes_kiq) { in mes_v11_0_sw_init()
1025 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs); in mes_v11_0_sw_fini()
1026 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs); in mes_v11_0_sw_fini()
1029 kfree(adev->mes.mqd_backup[pipe]); in mes_v11_0_sw_fini()
1031 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe], in mes_v11_0_sw_fini()
1032 &adev->mes.eop_gpu_addr[pipe], in mes_v11_0_sw_fini()
1034 amdgpu_ucode_release(&adev->mes.fw[pipe]); in mes_v11_0_sw_fini()
1037 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj, in mes_v11_0_sw_fini()
1038 &adev->gfx.kiq[0].ring.mqd_gpu_addr, in mes_v11_0_sw_fini()
1039 &adev->gfx.kiq[0].ring.mqd_ptr); in mes_v11_0_sw_fini()
1041 amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj, in mes_v11_0_sw_fini()
1042 &adev->mes.ring.mqd_gpu_addr, in mes_v11_0_sw_fini()
1043 &adev->mes.ring.mqd_ptr); in mes_v11_0_sw_fini()
1045 amdgpu_ring_fini(&adev->gfx.kiq[0].ring); in mes_v11_0_sw_fini()
1046 amdgpu_ring_fini(&adev->mes.ring); in mes_v11_0_sw_fini()
1048 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_sw_fini()
1061 struct amdgpu_device *adev = ring->adev; in mes_v11_0_kiq_dequeue()
1063 mutex_lock(&adev->srbm_mutex); in mes_v11_0_kiq_dequeue()
1064 soc21_grbm_select(adev, 3, ring->pipe, 0, 0); in mes_v11_0_kiq_dequeue()
1066 /* disable the queue if it's active */ in mes_v11_0_kiq_dequeue()
1069 for (i = 0; i < adev->usec_timeout; i++) { in mes_v11_0_kiq_dequeue()
1089 mutex_unlock(&adev->srbm_mutex); in mes_v11_0_kiq_dequeue()
1095 struct amdgpu_device *adev = ring->adev; in mes_v11_0_kiq_setting()
1100 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); in mes_v11_0_kiq_setting()
1120 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_kiq_hw_init()
1138 mes_v11_0_kiq_setting(&adev->gfx.kiq[0].ring); in mes_v11_0_kiq_hw_init()
1153 if (adev->mes.ring.sched.ready) { in mes_v11_0_kiq_hw_fini()
1154 mes_v11_0_kiq_dequeue(&adev->mes.ring); in mes_v11_0_kiq_hw_fini()
1155 adev->mes.ring.sched.ready = false; in mes_v11_0_kiq_hw_fini()
1159 mes_v11_0_kiq_dequeue(&adev->gfx.kiq[0].ring); in mes_v11_0_kiq_hw_fini()
1173 if (!adev->enable_mes_kiq) { in mes_v11_0_hw_init()
1174 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { in mes_v11_0_hw_init()
1190 r = mes_v11_0_set_hw_resources(&adev->mes); in mes_v11_0_hw_init()
1194 r = mes_v11_0_query_sched_status(&adev->mes); in mes_v11_0_hw_init()
1201 * Disable KIQ ring usage from the driver once MES is enabled. in mes_v11_0_hw_init()
1205 adev->gfx.kiq[0].ring.sched.ready = false; in mes_v11_0_hw_init()
1206 adev->mes.ring.sched.ready = true; in mes_v11_0_hw_init()
1250 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) in mes_v11_0_early_init()
1265 if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend && in mes_v11_0_late_init()