Lines Matching +full:gfx +full:- +full:mem
46 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_get()
49 offset = adev->doorbell_index.sdma_engine[0]; in amdgpu_mes_kernel_doorbell_get()
53 found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset); in amdgpu_mes_kernel_doorbell_get()
54 if (found >= mes->num_mes_dbs) { in amdgpu_mes_kernel_doorbell_get()
56 return -ENOSPC; in amdgpu_mes_kernel_doorbell_get()
59 set_bit(found, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_get()
62 *doorbell_index = mes->db_start_dw_offset + found * 2; in amdgpu_mes_kernel_doorbell_get()
70 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_free()
73 rel_index = (doorbell_index - mes->db_start_dw_offset) / 2; in amdgpu_mes_kernel_doorbell_free()
74 old = test_and_clear_bit(rel_index, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_free()
81 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_doorbell_init()
84 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); in amdgpu_mes_doorbell_init()
85 if (!mes->doorbell_bitmap) { in amdgpu_mes_doorbell_init()
87 return -ENOMEM; in amdgpu_mes_doorbell_init()
90 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; in amdgpu_mes_doorbell_init()
92 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; in amdgpu_mes_doorbell_init()
93 set_bit(i, mes->doorbell_bitmap); in amdgpu_mes_doorbell_init()
106 r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, in amdgpu_mes_event_log_init()
108 &adev->mes.event_log_gpu_obj, in amdgpu_mes_event_log_init()
109 &adev->mes.event_log_gpu_addr, in amdgpu_mes_event_log_init()
110 &adev->mes.event_log_cpu_addr); in amdgpu_mes_event_log_init()
112 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); in amdgpu_mes_event_log_init()
116 memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); in amdgpu_mes_event_log_init()
124 bitmap_free(adev->mes.doorbell_bitmap); in amdgpu_mes_doorbell_free()
131 adev->mes.adev = adev; in amdgpu_mes_init()
133 idr_init(&adev->mes.pasid_idr); in amdgpu_mes_init()
134 idr_init(&adev->mes.gang_id_idr); in amdgpu_mes_init()
135 idr_init(&adev->mes.queue_id_idr); in amdgpu_mes_init()
136 ida_init(&adev->mes.doorbell_ida); in amdgpu_mes_init()
137 spin_lock_init(&adev->mes.queue_id_lock); in amdgpu_mes_init()
138 mutex_init(&adev->mes.mutex_hidden); in amdgpu_mes_init()
141 spin_lock_init(&adev->mes.ring_lock[i]); in amdgpu_mes_init()
143 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; in amdgpu_mes_init()
144 adev->mes.vmid_mask_mmhub = 0xffffff00; in amdgpu_mes_init()
145 adev->mes.vmid_mask_gfxhub = 0xffffff00; in amdgpu_mes_init()
148 if (i >= (adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec)) in amdgpu_mes_init()
150 adev->mes.compute_hqd_mask[i] = 0xc; in amdgpu_mes_init()
154 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; in amdgpu_mes_init()
157 if (i >= adev->sdma.num_instances) in amdgpu_mes_init()
159 adev->mes.sdma_hqd_mask[i] = 0xfc; in amdgpu_mes_init()
163 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]); in amdgpu_mes_init()
165 dev_err(adev->dev, in amdgpu_mes_init()
170 adev->mes.sch_ctx_gpu_addr[i] = in amdgpu_mes_init()
171 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); in amdgpu_mes_init()
172 adev->mes.sch_ctx_ptr[i] = in amdgpu_mes_init()
173 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; in amdgpu_mes_init()
176 &adev->mes.query_status_fence_offs[i]); in amdgpu_mes_init()
178 dev_err(adev->dev, in amdgpu_mes_init()
183 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + in amdgpu_mes_init()
184 (adev->mes.query_status_fence_offs[i] * 4); in amdgpu_mes_init()
185 adev->mes.query_status_fence_ptr[i] = in amdgpu_mes_init()
186 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; in amdgpu_mes_init()
203 if (adev->mes.sch_ctx_ptr[i]) in amdgpu_mes_init()
204 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); in amdgpu_mes_init()
205 if (adev->mes.query_status_fence_ptr[i]) in amdgpu_mes_init()
207 adev->mes.query_status_fence_offs[i]); in amdgpu_mes_init()
210 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_init()
211 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_init()
212 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_init()
213 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_init()
214 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_init()
222 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, in amdgpu_mes_fini()
223 &adev->mes.event_log_gpu_addr, in amdgpu_mes_fini()
224 &adev->mes.event_log_cpu_addr); in amdgpu_mes_fini()
227 if (adev->mes.sch_ctx_ptr[i]) in amdgpu_mes_fini()
228 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); in amdgpu_mes_fini()
229 if (adev->mes.query_status_fence_ptr[i]) in amdgpu_mes_fini()
231 adev->mes.query_status_fence_offs[i]); in amdgpu_mes_fini()
236 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_fini()
237 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_fini()
238 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_fini()
239 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_fini()
240 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_fini()
245 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_free_mqd()
246 &q->mqd_gpu_addr, in amdgpu_mes_queue_free_mqd()
247 &q->mqd_cpu_ptr); in amdgpu_mes_queue_free_mqd()
260 return -ENOMEM; in amdgpu_mes_create_process()
266 &process->proc_ctx_bo, in amdgpu_mes_create_process()
267 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
268 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
273 memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); in amdgpu_mes_create_process()
279 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_create_process()
282 r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1, in amdgpu_mes_create_process()
289 INIT_LIST_HEAD(&process->gang_list); in amdgpu_mes_create_process()
290 process->vm = vm; in amdgpu_mes_create_process()
291 process->pasid = pasid; in amdgpu_mes_create_process()
292 process->process_quantum = adev->mes.default_process_quantum; in amdgpu_mes_create_process()
293 process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); in amdgpu_mes_create_process()
295 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
299 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
300 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_create_process()
301 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
302 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
321 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_destroy_process()
323 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
326 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
331 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
332 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
333 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
334 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_destroy_process()
335 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
337 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_destroy_process()
338 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_destroy_process()
340 r = adev->mes.funcs->remove_hw_queue(&adev->mes, in amdgpu_mes_destroy_process()
346 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_destroy_process()
349 idr_remove(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
350 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
353 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
355 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
357 list_del(&queue->list); in amdgpu_mes_destroy_process()
360 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_destroy_process()
361 &gang->gang_ctx_gpu_addr, in amdgpu_mes_destroy_process()
362 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
363 list_del(&gang->list); in amdgpu_mes_destroy_process()
367 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_destroy_process()
368 &process->proc_ctx_gpu_addr, in amdgpu_mes_destroy_process()
369 &process->proc_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
384 return -ENOMEM; in amdgpu_mes_add_gang()
390 &gang->gang_ctx_bo, in amdgpu_mes_add_gang()
391 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
392 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
397 memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); in amdgpu_mes_add_gang()
403 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_gang()
405 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_add_gang()
408 r = -EINVAL; in amdgpu_mes_add_gang()
413 r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0, in amdgpu_mes_add_gang()
420 gang->gang_id = r; in amdgpu_mes_add_gang()
423 INIT_LIST_HEAD(&gang->queue_list); in amdgpu_mes_add_gang()
424 gang->process = process; in amdgpu_mes_add_gang()
425 gang->priority = gprops->priority; in amdgpu_mes_add_gang()
426 gang->gang_quantum = gprops->gang_quantum ? in amdgpu_mes_add_gang()
427 gprops->gang_quantum : adev->mes.default_gang_quantum; in amdgpu_mes_add_gang()
428 gang->global_priority_level = gprops->global_priority_level; in amdgpu_mes_add_gang()
429 gang->inprocess_gang_priority = gprops->inprocess_gang_priority; in amdgpu_mes_add_gang()
430 list_add_tail(&gang->list, &process->gang_list); in amdgpu_mes_add_gang()
432 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
436 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
437 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_add_gang()
438 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
439 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
453 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_gang()
455 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_remove_gang()
458 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
459 return -EINVAL; in amdgpu_mes_remove_gang()
462 if (!list_empty(&gang->queue_list)) { in amdgpu_mes_remove_gang()
464 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
465 return -EBUSY; in amdgpu_mes_remove_gang()
468 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_remove_gang()
469 list_del(&gang->list); in amdgpu_mes_remove_gang()
470 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
472 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_remove_gang()
473 &gang->gang_ctx_gpu_addr, in amdgpu_mes_remove_gang()
474 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_remove_gang()
496 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_suspend()
497 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); in amdgpu_mes_suspend()
498 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_suspend()
520 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_resume()
521 r = adev->mes.funcs->resume_gang(&adev->mes, &input); in amdgpu_mes_resume()
522 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_resume()
533 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_alloc_mqd()
534 u32 mqd_size = mqd_mgr->mqd_size; in amdgpu_mes_queue_alloc_mqd()
539 &q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
540 &q->mqd_gpu_addr, &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
542 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r); in amdgpu_mes_queue_alloc_mqd()
545 memset(q->mqd_cpu_ptr, 0, mqd_size); in amdgpu_mes_queue_alloc_mqd()
547 r = amdgpu_bo_reserve(q->mqd_obj, false); in amdgpu_mes_queue_alloc_mqd()
554 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
555 &q->mqd_gpu_addr, in amdgpu_mes_queue_alloc_mqd()
556 &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
564 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_init_mqd()
567 mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr; in amdgpu_mes_queue_init_mqd()
568 mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr; in amdgpu_mes_queue_init_mqd()
569 mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
570 mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
571 mqd_prop.queue_size = p->queue_size; in amdgpu_mes_queue_init_mqd()
573 mqd_prop.doorbell_index = p->doorbell_off; in amdgpu_mes_queue_init_mqd()
574 mqd_prop.eop_gpu_addr = p->eop_gpu_addr; in amdgpu_mes_queue_init_mqd()
575 mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority; in amdgpu_mes_queue_init_mqd()
576 mqd_prop.hqd_queue_priority = p->hqd_queue_priority; in amdgpu_mes_queue_init_mqd()
579 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
580 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
581 mutex_lock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
582 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0); in amdgpu_mes_queue_init_mqd()
585 mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop); in amdgpu_mes_queue_init_mqd()
587 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
588 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
590 mutex_unlock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
593 amdgpu_bo_unreserve(q->mqd_obj); in amdgpu_mes_queue_init_mqd()
612 return -ENOMEM; in amdgpu_mes_add_hw_queue()
624 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_hw_queue()
626 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_hw_queue()
629 r = -EINVAL; in amdgpu_mes_add_hw_queue()
634 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
635 r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0, in amdgpu_mes_add_hw_queue()
638 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
641 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
642 *queue_id = queue->queue_id = r; in amdgpu_mes_add_hw_queue()
646 qprops->queue_type, in amdgpu_mes_add_hw_queue()
647 &qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
655 queue_input.process_id = gang->process->pasid; in amdgpu_mes_add_hw_queue()
658 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr - in amdgpu_mes_add_hw_queue()
659 adev->gmc.vram_start; in amdgpu_mes_add_hw_queue()
663 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; in amdgpu_mes_add_hw_queue()
664 queue_input.process_quantum = gang->process->process_quantum; in amdgpu_mes_add_hw_queue()
665 queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
666 queue_input.gang_quantum = gang->gang_quantum; in amdgpu_mes_add_hw_queue()
667 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
668 queue_input.inprocess_gang_priority = gang->inprocess_gang_priority; in amdgpu_mes_add_hw_queue()
669 queue_input.gang_global_priority_level = gang->global_priority_level; in amdgpu_mes_add_hw_queue()
670 queue_input.doorbell_offset = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
671 queue_input.mqd_addr = queue->mqd_gpu_addr; in amdgpu_mes_add_hw_queue()
672 queue_input.wptr_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
673 queue_input.wptr_mc_addr = qprops->wptr_mc_addr; in amdgpu_mes_add_hw_queue()
674 queue_input.queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
675 queue_input.paging = qprops->paging; in amdgpu_mes_add_hw_queue()
678 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_add_hw_queue()
681 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
687 gang->process->pasid, gang_id, qprops->queue_type, in amdgpu_mes_add_hw_queue()
688 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
690 queue->ring = qprops->ring; in amdgpu_mes_add_hw_queue()
691 queue->doorbell_off = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
692 queue->wptr_gpu_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
693 queue->queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
694 queue->paging = qprops->paging; in amdgpu_mes_add_hw_queue()
695 queue->gang = gang; in amdgpu_mes_add_hw_queue()
696 queue->ring->mqd_ptr = queue->mqd_cpu_ptr; in amdgpu_mes_add_hw_queue()
697 list_add_tail(&queue->list, &gang->queue_list); in amdgpu_mes_add_hw_queue()
699 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
703 amdgpu_mes_kernel_doorbell_free(adev, qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
705 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
706 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_add_hw_queue()
707 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
709 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
728 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_hw_queue()
731 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
733 queue = idr_find(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
735 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
736 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
738 return -EINVAL; in amdgpu_mes_remove_hw_queue()
741 idr_remove(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
742 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
745 queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
747 gang = queue->gang; in amdgpu_mes_remove_hw_queue()
748 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_remove_hw_queue()
749 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_remove_hw_queue()
751 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_remove_hw_queue()
756 list_del(&queue->list); in amdgpu_mes_remove_hw_queue()
757 amdgpu_mes_kernel_doorbell_free(adev, queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
758 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
777 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_reset_hw_queue()
780 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
782 queue = idr_find(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_reset_hw_queue()
784 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
785 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_reset_hw_queue()
787 return -EINVAL; in amdgpu_mes_reset_hw_queue()
789 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
792 queue->doorbell_off); in amdgpu_mes_reset_hw_queue()
794 gang = queue->gang; in amdgpu_mes_reset_hw_queue()
795 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_reset_hw_queue()
796 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_reset_hw_queue()
798 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_hw_queue()
803 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_reset_hw_queue()
820 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_hw_queue_mmio()
835 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_map_legacy_queue()
836 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_map_legacy_queue()
837 queue_input.pipe_id = ring->pipe; in amdgpu_mes_map_legacy_queue()
838 queue_input.queue_id = ring->queue; in amdgpu_mes_map_legacy_queue()
839 queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); in amdgpu_mes_map_legacy_queue()
840 queue_input.wptr_addr = ring->wptr_gpu_addr; in amdgpu_mes_map_legacy_queue()
842 r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_map_legacy_queue()
858 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_unmap_legacy_queue()
859 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_unmap_legacy_queue()
860 queue_input.pipe_id = ring->pipe; in amdgpu_mes_unmap_legacy_queue()
861 queue_input.queue_id = ring->queue; in amdgpu_mes_unmap_legacy_queue()
865 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_unmap_legacy_queue()
882 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_reset_legacy_queue()
883 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_reset_legacy_queue()
884 queue_input.me_id = ring->me; in amdgpu_mes_reset_legacy_queue()
885 queue_input.pipe_id = ring->pipe; in amdgpu_mes_reset_legacy_queue()
886 queue_input.queue_id = ring->queue; in amdgpu_mes_reset_legacy_queue()
887 queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; in amdgpu_mes_reset_legacy_queue()
888 queue_input.wptr_addr = ring->wptr_gpu_addr; in amdgpu_mes_reset_legacy_queue()
892 r = adev->mes.funcs->reset_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_legacy_queue()
911 read_val_gpu_addr = adev->wb.gpu_addr + (addr_offset * 4); in amdgpu_mes_rreg()
912 read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; in amdgpu_mes_rreg()
917 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_rreg()
922 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_rreg()
944 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_wreg()
946 r = -EINVAL; in amdgpu_mes_wreg()
950 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_wreg()
971 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_write_reg_wait()
973 r = -EINVAL; in amdgpu_mes_reg_write_reg_wait()
977 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_write_reg_wait()
996 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_wait()
998 r = -EINVAL; in amdgpu_mes_reg_wait()
1002 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_wait()
1020 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_set_shader_debugger()
1022 return -EINVAL; in amdgpu_mes_set_shader_debugger()
1031 return -EINVAL; in amdgpu_mes_set_shader_debugger()
1037 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> in amdgpu_mes_set_shader_debugger()
1041 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_set_shader_debugger()
1043 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_set_shader_debugger()
1047 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_set_shader_debugger()
1058 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_flush_shader_debugger()
1060 return -EINVAL; in amdgpu_mes_flush_shader_debugger()
1067 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
1069 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_flush_shader_debugger()
1073 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
1083 props->queue_type = ring->funcs->type; in amdgpu_mes_ring_to_queue_props()
1084 props->hqd_base_gpu_addr = ring->gpu_addr; in amdgpu_mes_ring_to_queue_props()
1085 props->rptr_gpu_addr = ring->rptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1086 props->wptr_gpu_addr = ring->wptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1087 props->wptr_mc_addr = in amdgpu_mes_ring_to_queue_props()
1088 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs; in amdgpu_mes_ring_to_queue_props()
1089 props->queue_size = ring->ring_size; in amdgpu_mes_ring_to_queue_props()
1090 props->eop_gpu_addr = ring->eop_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1091 props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; in amdgpu_mes_ring_to_queue_props()
1092 props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; in amdgpu_mes_ring_to_queue_props()
1093 props->paging = false; in amdgpu_mes_ring_to_queue_props()
1094 props->ring = ring; in amdgpu_mes_ring_to_queue_props()
1101 _eng[ring->idx].slots[id_offs]); \
1104 _eng[ring->idx].ring); \
1107 _eng[ring->idx].ib); \
1110 _eng[ring->idx].padding); \
1115 switch (ring->funcs->type) { in amdgpu_mes_ctx_get_offs()
1117 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx); in amdgpu_mes_ctx_get_offs()
1130 return -EINVAL; in amdgpu_mes_ctx_get_offs()
1147 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_ring()
1148 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_ring()
1151 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1152 return -EINVAL; in amdgpu_mes_add_ring()
1154 pasid = gang->process->pasid; in amdgpu_mes_add_ring()
1158 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1159 return -ENOMEM; in amdgpu_mes_add_ring()
1162 ring->ring_obj = NULL; in amdgpu_mes_add_ring()
1163 ring->use_doorbell = true; in amdgpu_mes_add_ring()
1164 ring->is_mes_queue = true; in amdgpu_mes_add_ring()
1165 ring->mes_ctx = ctx_data; in amdgpu_mes_add_ring()
1166 ring->idx = idx; in amdgpu_mes_add_ring()
1167 ring->no_scheduler = true; in amdgpu_mes_add_ring()
1171 compute[ring->idx].mec_hpd); in amdgpu_mes_add_ring()
1172 ring->eop_gpu_addr = in amdgpu_mes_add_ring()
1178 ring->funcs = adev->gfx.gfx_ring[0].funcs; in amdgpu_mes_add_ring()
1179 ring->me = adev->gfx.gfx_ring[0].me; in amdgpu_mes_add_ring()
1180 ring->pipe = adev->gfx.gfx_ring[0].pipe; in amdgpu_mes_add_ring()
1183 ring->funcs = adev->gfx.compute_ring[0].funcs; in amdgpu_mes_add_ring()
1184 ring->me = adev->gfx.compute_ring[0].me; in amdgpu_mes_add_ring()
1185 ring->pipe = adev->gfx.compute_ring[0].pipe; in amdgpu_mes_add_ring()
1188 ring->funcs = adev->sdma.instance[0].ring.funcs; in amdgpu_mes_add_ring()
1197 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1203 dma_fence_wait(gang->process->vm->last_update, false); in amdgpu_mes_add_ring()
1204 dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false); in amdgpu_mes_add_ring()
1205 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1211 ring->hw_queue_id = queue_id; in amdgpu_mes_add_ring()
1212 ring->doorbell_index = qprops.doorbell_off; in amdgpu_mes_add_ring()
1215 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id); in amdgpu_mes_add_ring()
1217 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1220 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1241 amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id); in amdgpu_mes_remove_ring()
1242 timer_delete_sync(&ring->fence_drv.fallback_timer); in amdgpu_mes_remove_ring()
1250 return adev->mes.aggregated_doorbells[prio]; in amdgpu_mes_get_aggregated_doorbell_index()
1261 &ctx_data->meta_data_obj, in amdgpu_mes_ctx_alloc_meta_data()
1262 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_alloc_meta_data()
1263 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_alloc_meta_data()
1265 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r); in amdgpu_mes_ctx_alloc_meta_data()
1269 if (!ctx_data->meta_data_obj) in amdgpu_mes_ctx_alloc_meta_data()
1270 return -ENOMEM; in amdgpu_mes_ctx_alloc_meta_data()
1272 memset(ctx_data->meta_data_ptr, 0, in amdgpu_mes_ctx_alloc_meta_data()
1280 if (ctx_data->meta_data_obj) in amdgpu_mes_ctx_free_meta_data()
1281 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj, in amdgpu_mes_ctx_free_meta_data()
1282 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_free_meta_data()
1283 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_free_meta_data()
1300 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_map_meta_data()
1311 bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj); in amdgpu_mes_ctx_map_meta_data()
1314 r = -ENOMEM; in amdgpu_mes_ctx_map_meta_data()
1318 r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0, in amdgpu_mes_ctx_map_meta_data()
1333 amdgpu_sync_fence(&sync, bo_va->last_pt_update, GFP_KERNEL); in amdgpu_mes_ctx_map_meta_data()
1340 amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL); in amdgpu_mes_ctx_map_meta_data()
1346 ctx_data->meta_data_va = bo_va; in amdgpu_mes_ctx_map_meta_data()
1361 struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va; in amdgpu_mes_ctx_unmap_meta_data()
1362 struct amdgpu_bo *bo = ctx_data->meta_data_obj; in amdgpu_mes_ctx_unmap_meta_data()
1363 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_mes_ctx_unmap_meta_data()
1371 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_unmap_meta_data()
1386 r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, in amdgpu_mes_ctx_unmap_meta_data()
1405 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r); in amdgpu_mes_ctx_unmap_meta_data()
1423 gprops.gang_quantum = adev->mes.default_gang_quantum; in amdgpu_mes_test_create_gang_and_queues()
1443 DRM_INFO("ring %s was added\n", ring->name); in amdgpu_mes_test_create_gang_and_queues()
1466 DRM_DEV_ERROR(ring->adev->dev, in amdgpu_mes_test_queues()
1468 ring->name, r); in amdgpu_mes_test_queues()
1471 DRM_INFO("ring %s ib test pass\n", ring->name); in amdgpu_mes_test_queues()
1490 dev_warn(adev->dev, "No more PASIDs available!"); in amdgpu_mes_self_test()
1496 r = -ENOMEM; in amdgpu_mes_self_test()
1500 r = amdgpu_vm_init(adev, vm, -1); in amdgpu_mes_self_test()
1526 /* On GFX v10.3, fw hasn't supported to map sdma queue. */ in amdgpu_mes_self_test()
1592 if (adev->enable_uni_mes) { in amdgpu_mes_init_microcode()
1607 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED, in amdgpu_mes_init_microcode()
1610 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix); in amdgpu_mes_init_microcode()
1611 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], in amdgpu_mes_init_microcode()
1620 adev->mes.fw[pipe]->data; in amdgpu_mes_init_microcode()
1621 adev->mes.uc_start_addr[pipe] = in amdgpu_mes_init_microcode()
1622 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | in amdgpu_mes_init_microcode()
1623 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1624 adev->mes.data_start_addr[pipe] = in amdgpu_mes_init_microcode()
1625 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | in amdgpu_mes_init_microcode()
1626 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1627 ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data + in amdgpu_mes_init_microcode()
1629 adev->mes.fw_version[pipe] = in amdgpu_mes_init_microcode()
1632 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { in amdgpu_mes_init_microcode()
1643 info = &adev->firmware.ucode[ucode]; in amdgpu_mes_init_microcode()
1644 info->ucode_id = ucode; in amdgpu_mes_init_microcode()
1645 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1646 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1647 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), in amdgpu_mes_init_microcode()
1650 info = &adev->firmware.ucode[ucode_data]; in amdgpu_mes_init_microcode()
1651 info->ucode_id = ucode_data; in amdgpu_mes_init_microcode()
1652 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1653 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1654 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), in amdgpu_mes_init_microcode()
1660 amdgpu_ucode_release(&adev->mes.fw[pipe]); in amdgpu_mes_init_microcode()
1666 uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; in amdgpu_mes_suspend_resume_all_supported()
1677 /* Fix me -- node_id is used to identify the correct MES instances in the future */
1687 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_set_enforce_isolation()
1688 dev_err(adev->dev, "mes change config is not supported!\n"); in amdgpu_mes_set_enforce_isolation()
1689 r = -EINVAL; in amdgpu_mes_set_enforce_isolation()
1693 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_set_enforce_isolation()
1695 dev_err(adev->dev, "failed to change_config.\n"); in amdgpu_mes_set_enforce_isolation()
1705 if (adev->enable_mes && adev->gfx.enable_cleaner_shader) { in amdgpu_mes_update_enforce_isolation()
1706 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_mes_update_enforce_isolation()
1707 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { in amdgpu_mes_update_enforce_isolation()
1708 if (adev->enforce_isolation[i]) in amdgpu_mes_update_enforce_isolation()
1713 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_mes_update_enforce_isolation()
1722 struct amdgpu_device *adev = m->private; in amdgpu_debugfs_mes_event_log_show()
1723 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); in amdgpu_debugfs_mes_event_log_show() local
1726 mem, adev->mes.event_log_size, false); in amdgpu_debugfs_mes_event_log_show()
1739 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_debugfs_mes_event_log_init()
1740 struct dentry *root = minor->debugfs_root; in amdgpu_debugfs_mes_event_log_init()
1741 if (adev->enable_mes && amdgpu_mes_log_enable) in amdgpu_debugfs_mes_event_log_init()