Lines Matching +full:gfx +full:- +full:mem
47 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_get()
50 offset = adev->doorbell_index.sdma_engine[0]; in amdgpu_mes_kernel_doorbell_get()
54 found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset); in amdgpu_mes_kernel_doorbell_get()
55 if (found >= mes->num_mes_dbs) { in amdgpu_mes_kernel_doorbell_get()
57 return -ENOSPC; in amdgpu_mes_kernel_doorbell_get()
60 set_bit(found, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_get()
63 *doorbell_index = mes->db_start_dw_offset + found * 2; in amdgpu_mes_kernel_doorbell_get()
72 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_free()
75 rel_index = (doorbell_index - mes->db_start_dw_offset) / 2; in amdgpu_mes_kernel_doorbell_free()
76 old = test_and_clear_bit(rel_index, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_free()
83 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_doorbell_init()
86 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); in amdgpu_mes_doorbell_init()
87 if (!mes->doorbell_bitmap) { in amdgpu_mes_doorbell_init()
89 return -ENOMEM; in amdgpu_mes_doorbell_init()
92 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; in amdgpu_mes_doorbell_init()
94 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; in amdgpu_mes_doorbell_init()
95 set_bit(i, mes->doorbell_bitmap); in amdgpu_mes_doorbell_init()
107 &adev->mes.event_log_gpu_obj, in amdgpu_mes_event_log_init()
108 &adev->mes.event_log_gpu_addr, in amdgpu_mes_event_log_init()
109 &adev->mes.event_log_cpu_addr); in amdgpu_mes_event_log_init()
111 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); in amdgpu_mes_event_log_init()
115 memset(adev->mes.event_log_cpu_addr, 0, PAGE_SIZE); in amdgpu_mes_event_log_init()
123 bitmap_free(adev->mes.doorbell_bitmap); in amdgpu_mes_doorbell_free()
130 adev->mes.adev = adev; in amdgpu_mes_init()
132 idr_init(&adev->mes.pasid_idr); in amdgpu_mes_init()
133 idr_init(&adev->mes.gang_id_idr); in amdgpu_mes_init()
134 idr_init(&adev->mes.queue_id_idr); in amdgpu_mes_init()
135 ida_init(&adev->mes.doorbell_ida); in amdgpu_mes_init()
136 spin_lock_init(&adev->mes.queue_id_lock); in amdgpu_mes_init()
137 spin_lock_init(&adev->mes.ring_lock); in amdgpu_mes_init()
138 mutex_init(&adev->mes.mutex_hidden); in amdgpu_mes_init()
140 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; in amdgpu_mes_init()
141 adev->mes.vmid_mask_mmhub = 0xffffff00; in amdgpu_mes_init()
142 adev->mes.vmid_mask_gfxhub = 0xffffff00; in amdgpu_mes_init()
148 adev->mes.compute_hqd_mask[i] = 0xc; in amdgpu_mes_init()
152 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; in amdgpu_mes_init()
157 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc; in amdgpu_mes_init()
158 /* zero sdma_hqd_mask for non-existent engine */ in amdgpu_mes_init()
159 else if (adev->sdma.num_instances == 1) in amdgpu_mes_init()
160 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc; in amdgpu_mes_init()
162 adev->mes.sdma_hqd_mask[i] = 0xfc; in amdgpu_mes_init()
165 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs); in amdgpu_mes_init()
167 dev_err(adev->dev, in amdgpu_mes_init()
171 adev->mes.sch_ctx_gpu_addr = in amdgpu_mes_init()
172 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4); in amdgpu_mes_init()
173 adev->mes.sch_ctx_ptr = in amdgpu_mes_init()
174 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs]; in amdgpu_mes_init()
176 r = amdgpu_device_wb_get(adev, &adev->mes.query_status_fence_offs); in amdgpu_mes_init()
178 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs); in amdgpu_mes_init()
179 dev_err(adev->dev, in amdgpu_mes_init()
183 adev->mes.query_status_fence_gpu_addr = in amdgpu_mes_init()
184 adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4); in amdgpu_mes_init()
185 adev->mes.query_status_fence_ptr = in amdgpu_mes_init()
186 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs]; in amdgpu_mes_init()
188 r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs); in amdgpu_mes_init()
190 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs); in amdgpu_mes_init()
191 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs); in amdgpu_mes_init()
192 dev_err(adev->dev, in amdgpu_mes_init()
196 adev->mes.read_val_gpu_addr = in amdgpu_mes_init()
197 adev->wb.gpu_addr + (adev->mes.read_val_offs * 4); in amdgpu_mes_init()
198 adev->mes.read_val_ptr = in amdgpu_mes_init()
199 (uint32_t *)&adev->wb.wb[adev->mes.read_val_offs]; in amdgpu_mes_init()
214 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs); in amdgpu_mes_init()
215 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs); in amdgpu_mes_init()
216 amdgpu_device_wb_free(adev, adev->mes.read_val_offs); in amdgpu_mes_init()
218 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_init()
219 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_init()
220 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_init()
221 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_init()
222 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_init()
228 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, in amdgpu_mes_fini()
229 &adev->mes.event_log_gpu_addr, in amdgpu_mes_fini()
230 &adev->mes.event_log_cpu_addr); in amdgpu_mes_fini()
232 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs); in amdgpu_mes_fini()
233 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs); in amdgpu_mes_fini()
234 amdgpu_device_wb_free(adev, adev->mes.read_val_offs); in amdgpu_mes_fini()
237 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_fini()
238 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_fini()
239 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_fini()
240 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_fini()
241 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_fini()
246 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_free_mqd()
247 &q->mqd_gpu_addr, in amdgpu_mes_queue_free_mqd()
248 &q->mqd_cpu_ptr); in amdgpu_mes_queue_free_mqd()
261 return -ENOMEM; in amdgpu_mes_create_process()
267 &process->proc_ctx_bo, in amdgpu_mes_create_process()
268 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
269 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
274 memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); in amdgpu_mes_create_process()
280 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_create_process()
283 r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1, in amdgpu_mes_create_process()
290 INIT_LIST_HEAD(&process->gang_list); in amdgpu_mes_create_process()
291 process->vm = vm; in amdgpu_mes_create_process()
292 process->pasid = pasid; in amdgpu_mes_create_process()
293 process->process_quantum = adev->mes.default_process_quantum; in amdgpu_mes_create_process()
294 process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); in amdgpu_mes_create_process()
296 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
300 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
301 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_create_process()
302 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
303 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
322 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_destroy_process()
324 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
327 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
332 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
333 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
334 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
335 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_destroy_process()
336 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
338 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_destroy_process()
339 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_destroy_process()
341 r = adev->mes.funcs->remove_hw_queue(&adev->mes, in amdgpu_mes_destroy_process()
347 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_destroy_process()
350 idr_remove(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
351 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
354 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
356 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
358 list_del(&queue->list); in amdgpu_mes_destroy_process()
361 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_destroy_process()
362 &gang->gang_ctx_gpu_addr, in amdgpu_mes_destroy_process()
363 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
364 list_del(&gang->list); in amdgpu_mes_destroy_process()
368 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_destroy_process()
369 &process->proc_ctx_gpu_addr, in amdgpu_mes_destroy_process()
370 &process->proc_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
385 return -ENOMEM; in amdgpu_mes_add_gang()
391 &gang->gang_ctx_bo, in amdgpu_mes_add_gang()
392 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
393 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
398 memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); in amdgpu_mes_add_gang()
404 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_gang()
406 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_add_gang()
409 r = -EINVAL; in amdgpu_mes_add_gang()
414 r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0, in amdgpu_mes_add_gang()
421 gang->gang_id = r; in amdgpu_mes_add_gang()
424 INIT_LIST_HEAD(&gang->queue_list); in amdgpu_mes_add_gang()
425 gang->process = process; in amdgpu_mes_add_gang()
426 gang->priority = gprops->priority; in amdgpu_mes_add_gang()
427 gang->gang_quantum = gprops->gang_quantum ? in amdgpu_mes_add_gang()
428 gprops->gang_quantum : adev->mes.default_gang_quantum; in amdgpu_mes_add_gang()
429 gang->global_priority_level = gprops->global_priority_level; in amdgpu_mes_add_gang()
430 gang->inprocess_gang_priority = gprops->inprocess_gang_priority; in amdgpu_mes_add_gang()
431 list_add_tail(&gang->list, &process->gang_list); in amdgpu_mes_add_gang()
433 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
437 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
438 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_add_gang()
439 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
440 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
454 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_gang()
456 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_remove_gang()
459 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
460 return -EINVAL; in amdgpu_mes_remove_gang()
463 if (!list_empty(&gang->queue_list)) { in amdgpu_mes_remove_gang()
465 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
466 return -EBUSY; in amdgpu_mes_remove_gang()
469 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_remove_gang()
470 list_del(&gang->list); in amdgpu_mes_remove_gang()
471 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
473 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_remove_gang()
474 &gang->gang_ctx_gpu_addr, in amdgpu_mes_remove_gang()
475 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_remove_gang()
494 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_suspend()
496 idp = &adev->mes.pasid_idr; in amdgpu_mes_suspend()
499 list_for_each_entry(gang, &process->gang_list, list) { in amdgpu_mes_suspend()
500 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); in amdgpu_mes_suspend()
503 pasid, gang->gang_id); in amdgpu_mes_suspend()
507 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_suspend()
523 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_resume()
525 idp = &adev->mes.pasid_idr; in amdgpu_mes_resume()
528 list_for_each_entry(gang, &process->gang_list, list) { in amdgpu_mes_resume()
529 r = adev->mes.funcs->resume_gang(&adev->mes, &input); in amdgpu_mes_resume()
532 pasid, gang->gang_id); in amdgpu_mes_resume()
536 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_resume()
544 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_alloc_mqd()
545 u32 mqd_size = mqd_mgr->mqd_size; in amdgpu_mes_queue_alloc_mqd()
550 &q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
551 &q->mqd_gpu_addr, &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
553 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r); in amdgpu_mes_queue_alloc_mqd()
556 memset(q->mqd_cpu_ptr, 0, mqd_size); in amdgpu_mes_queue_alloc_mqd()
558 r = amdgpu_bo_reserve(q->mqd_obj, false); in amdgpu_mes_queue_alloc_mqd()
565 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
566 &q->mqd_gpu_addr, in amdgpu_mes_queue_alloc_mqd()
567 &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
575 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_init_mqd()
578 mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr; in amdgpu_mes_queue_init_mqd()
579 mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr; in amdgpu_mes_queue_init_mqd()
580 mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
581 mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
582 mqd_prop.queue_size = p->queue_size; in amdgpu_mes_queue_init_mqd()
584 mqd_prop.doorbell_index = p->doorbell_off; in amdgpu_mes_queue_init_mqd()
585 mqd_prop.eop_gpu_addr = p->eop_gpu_addr; in amdgpu_mes_queue_init_mqd()
586 mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority; in amdgpu_mes_queue_init_mqd()
587 mqd_prop.hqd_queue_priority = p->hqd_queue_priority; in amdgpu_mes_queue_init_mqd()
590 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
591 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
592 mutex_lock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
593 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0); in amdgpu_mes_queue_init_mqd()
596 mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop); in amdgpu_mes_queue_init_mqd()
598 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
599 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
601 mutex_unlock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
604 amdgpu_bo_unreserve(q->mqd_obj); in amdgpu_mes_queue_init_mqd()
623 return -ENOMEM; in amdgpu_mes_add_hw_queue()
635 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_hw_queue()
637 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_hw_queue()
640 r = -EINVAL; in amdgpu_mes_add_hw_queue()
645 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
646 r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0, in amdgpu_mes_add_hw_queue()
649 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
652 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
653 *queue_id = queue->queue_id = r; in amdgpu_mes_add_hw_queue()
656 r = amdgpu_mes_kernel_doorbell_get(adev, gang->process, in amdgpu_mes_add_hw_queue()
657 qprops->queue_type, in amdgpu_mes_add_hw_queue()
658 &qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
666 queue_input.process_id = gang->process->pasid; in amdgpu_mes_add_hw_queue()
669 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr - in amdgpu_mes_add_hw_queue()
670 adev->gmc.vram_start; in amdgpu_mes_add_hw_queue()
674 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; in amdgpu_mes_add_hw_queue()
675 queue_input.process_quantum = gang->process->process_quantum; in amdgpu_mes_add_hw_queue()
676 queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
677 queue_input.gang_quantum = gang->gang_quantum; in amdgpu_mes_add_hw_queue()
678 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
679 queue_input.inprocess_gang_priority = gang->inprocess_gang_priority; in amdgpu_mes_add_hw_queue()
680 queue_input.gang_global_priority_level = gang->global_priority_level; in amdgpu_mes_add_hw_queue()
681 queue_input.doorbell_offset = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
682 queue_input.mqd_addr = queue->mqd_gpu_addr; in amdgpu_mes_add_hw_queue()
683 queue_input.wptr_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
684 queue_input.wptr_mc_addr = qprops->wptr_mc_addr; in amdgpu_mes_add_hw_queue()
685 queue_input.queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
686 queue_input.paging = qprops->paging; in amdgpu_mes_add_hw_queue()
689 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_add_hw_queue()
692 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
698 gang->process->pasid, gang_id, qprops->queue_type, in amdgpu_mes_add_hw_queue()
699 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
701 queue->ring = qprops->ring; in amdgpu_mes_add_hw_queue()
702 queue->doorbell_off = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
703 queue->wptr_gpu_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
704 queue->queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
705 queue->paging = qprops->paging; in amdgpu_mes_add_hw_queue()
706 queue->gang = gang; in amdgpu_mes_add_hw_queue()
707 queue->ring->mqd_ptr = queue->mqd_cpu_ptr; in amdgpu_mes_add_hw_queue()
708 list_add_tail(&queue->list, &gang->queue_list); in amdgpu_mes_add_hw_queue()
710 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
714 amdgpu_mes_kernel_doorbell_free(adev, gang->process, in amdgpu_mes_add_hw_queue()
715 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
717 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
718 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_add_hw_queue()
719 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
721 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
740 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_hw_queue()
743 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
745 queue = idr_find(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
747 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
748 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
750 return -EINVAL; in amdgpu_mes_remove_hw_queue()
753 idr_remove(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
754 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
757 queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
759 gang = queue->gang; in amdgpu_mes_remove_hw_queue()
760 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_remove_hw_queue()
761 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_remove_hw_queue()
763 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_remove_hw_queue()
768 list_del(&queue->list); in amdgpu_mes_remove_hw_queue()
769 amdgpu_mes_kernel_doorbell_free(adev, gang->process, in amdgpu_mes_remove_hw_queue()
770 queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
771 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
787 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_unmap_legacy_queue()
788 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_unmap_legacy_queue()
789 queue_input.pipe_id = ring->pipe; in amdgpu_mes_unmap_legacy_queue()
790 queue_input.queue_id = ring->queue; in amdgpu_mes_unmap_legacy_queue()
794 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_unmap_legacy_queue()
808 op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr; in amdgpu_mes_rreg()
810 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_rreg()
815 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_rreg()
819 val = *(adev->mes.read_val_ptr); in amdgpu_mes_rreg()
835 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_wreg()
837 r = -EINVAL; in amdgpu_mes_wreg()
841 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_wreg()
862 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_write_reg_wait()
864 r = -EINVAL; in amdgpu_mes_reg_write_reg_wait()
868 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_write_reg_wait()
887 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_wait()
889 r = -EINVAL; in amdgpu_mes_reg_wait()
893 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_wait()
911 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_set_shader_debugger()
913 return -EINVAL; in amdgpu_mes_set_shader_debugger()
922 return -EINVAL; in amdgpu_mes_set_shader_debugger()
928 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> in amdgpu_mes_set_shader_debugger()
932 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_set_shader_debugger()
934 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_set_shader_debugger()
938 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_set_shader_debugger()
949 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_flush_shader_debugger()
951 return -EINVAL; in amdgpu_mes_flush_shader_debugger()
958 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
960 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_flush_shader_debugger()
964 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
974 props->queue_type = ring->funcs->type; in amdgpu_mes_ring_to_queue_props()
975 props->hqd_base_gpu_addr = ring->gpu_addr; in amdgpu_mes_ring_to_queue_props()
976 props->rptr_gpu_addr = ring->rptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
977 props->wptr_gpu_addr = ring->wptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
978 props->wptr_mc_addr = in amdgpu_mes_ring_to_queue_props()
979 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs; in amdgpu_mes_ring_to_queue_props()
980 props->queue_size = ring->ring_size; in amdgpu_mes_ring_to_queue_props()
981 props->eop_gpu_addr = ring->eop_gpu_addr; in amdgpu_mes_ring_to_queue_props()
982 props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; in amdgpu_mes_ring_to_queue_props()
983 props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; in amdgpu_mes_ring_to_queue_props()
984 props->paging = false; in amdgpu_mes_ring_to_queue_props()
985 props->ring = ring; in amdgpu_mes_ring_to_queue_props()
992 _eng[ring->idx].slots[id_offs]); \
995 _eng[ring->idx].ring); \
998 _eng[ring->idx].ib); \
1001 _eng[ring->idx].padding); \
1006 switch (ring->funcs->type) { in amdgpu_mes_ctx_get_offs()
1008 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx); in amdgpu_mes_ctx_get_offs()
1021 return -EINVAL; in amdgpu_mes_ctx_get_offs()
1038 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_ring()
1039 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_ring()
1042 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1043 return -EINVAL; in amdgpu_mes_add_ring()
1045 pasid = gang->process->pasid; in amdgpu_mes_add_ring()
1049 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1050 return -ENOMEM; in amdgpu_mes_add_ring()
1053 ring->ring_obj = NULL; in amdgpu_mes_add_ring()
1054 ring->use_doorbell = true; in amdgpu_mes_add_ring()
1055 ring->is_mes_queue = true; in amdgpu_mes_add_ring()
1056 ring->mes_ctx = ctx_data; in amdgpu_mes_add_ring()
1057 ring->idx = idx; in amdgpu_mes_add_ring()
1058 ring->no_scheduler = true; in amdgpu_mes_add_ring()
1062 compute[ring->idx].mec_hpd); in amdgpu_mes_add_ring()
1063 ring->eop_gpu_addr = in amdgpu_mes_add_ring()
1069 ring->funcs = adev->gfx.gfx_ring[0].funcs; in amdgpu_mes_add_ring()
1070 ring->me = adev->gfx.gfx_ring[0].me; in amdgpu_mes_add_ring()
1071 ring->pipe = adev->gfx.gfx_ring[0].pipe; in amdgpu_mes_add_ring()
1074 ring->funcs = adev->gfx.compute_ring[0].funcs; in amdgpu_mes_add_ring()
1075 ring->me = adev->gfx.compute_ring[0].me; in amdgpu_mes_add_ring()
1076 ring->pipe = adev->gfx.compute_ring[0].pipe; in amdgpu_mes_add_ring()
1079 ring->funcs = adev->sdma.instance[0].ring.funcs; in amdgpu_mes_add_ring()
1092 dma_fence_wait(gang->process->vm->last_update, false); in amdgpu_mes_add_ring()
1093 dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false); in amdgpu_mes_add_ring()
1094 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1100 ring->hw_queue_id = queue_id; in amdgpu_mes_add_ring()
1101 ring->doorbell_index = qprops.doorbell_off; in amdgpu_mes_add_ring()
1104 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id); in amdgpu_mes_add_ring()
1106 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1109 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1121 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1131 amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id); in amdgpu_mes_remove_ring()
1139 return adev->mes.aggregated_doorbells[prio]; in amdgpu_mes_get_aggregated_doorbell_index()
1150 &ctx_data->meta_data_obj, in amdgpu_mes_ctx_alloc_meta_data()
1151 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_alloc_meta_data()
1152 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_alloc_meta_data()
1154 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r); in amdgpu_mes_ctx_alloc_meta_data()
1158 if (!ctx_data->meta_data_obj) in amdgpu_mes_ctx_alloc_meta_data()
1159 return -ENOMEM; in amdgpu_mes_ctx_alloc_meta_data()
1161 memset(ctx_data->meta_data_ptr, 0, in amdgpu_mes_ctx_alloc_meta_data()
1169 if (ctx_data->meta_data_obj) in amdgpu_mes_ctx_free_meta_data()
1170 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj, in amdgpu_mes_ctx_free_meta_data()
1171 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_free_meta_data()
1172 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_free_meta_data()
1189 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_map_meta_data()
1200 bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj); in amdgpu_mes_ctx_map_meta_data()
1203 r = -ENOMEM; in amdgpu_mes_ctx_map_meta_data()
1207 r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0, in amdgpu_mes_ctx_map_meta_data()
1222 amdgpu_sync_fence(&sync, bo_va->last_pt_update); in amdgpu_mes_ctx_map_meta_data()
1229 amdgpu_sync_fence(&sync, vm->last_update); in amdgpu_mes_ctx_map_meta_data()
1235 ctx_data->meta_data_va = bo_va; in amdgpu_mes_ctx_map_meta_data()
1250 struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va; in amdgpu_mes_ctx_unmap_meta_data()
1251 struct amdgpu_bo *bo = ctx_data->meta_data_obj; in amdgpu_mes_ctx_unmap_meta_data()
1252 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_mes_ctx_unmap_meta_data()
1260 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_unmap_meta_data()
1275 r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, in amdgpu_mes_ctx_unmap_meta_data()
1294 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r); in amdgpu_mes_ctx_unmap_meta_data()
1312 gprops.gang_quantum = adev->mes.default_gang_quantum; in amdgpu_mes_test_create_gang_and_queues()
1332 DRM_INFO("ring %s was added\n", ring->name); in amdgpu_mes_test_create_gang_and_queues()
1355 DRM_DEV_ERROR(ring->adev->dev, in amdgpu_mes_test_queues()
1357 ring->name, r); in amdgpu_mes_test_queues()
1360 DRM_INFO("ring %s ib test pass\n", ring->name); in amdgpu_mes_test_queues()
1379 dev_warn(adev->dev, "No more PASIDs available!"); in amdgpu_mes_self_test()
1385 r = -ENOMEM; in amdgpu_mes_self_test()
1389 r = amdgpu_vm_init(adev, vm, -1); in amdgpu_mes_self_test()
1415 /* On GFX v10.3, fw hasn't supported to map sdma queue. */ in amdgpu_mes_self_test()
1491 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name); in amdgpu_mes_init_microcode()
1496 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], in amdgpu_mes_init_microcode()
1504 adev->mes.fw[pipe]->data; in amdgpu_mes_init_microcode()
1505 adev->mes.uc_start_addr[pipe] = in amdgpu_mes_init_microcode()
1506 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | in amdgpu_mes_init_microcode()
1507 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1508 adev->mes.data_start_addr[pipe] = in amdgpu_mes_init_microcode()
1509 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | in amdgpu_mes_init_microcode()
1510 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1512 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { in amdgpu_mes_init_microcode()
1523 info = &adev->firmware.ucode[ucode]; in amdgpu_mes_init_microcode()
1524 info->ucode_id = ucode; in amdgpu_mes_init_microcode()
1525 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1526 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1527 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), in amdgpu_mes_init_microcode()
1530 info = &adev->firmware.ucode[ucode_data]; in amdgpu_mes_init_microcode()
1531 info->ucode_id = ucode_data; in amdgpu_mes_init_microcode()
1532 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1533 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1534 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), in amdgpu_mes_init_microcode()
1540 amdgpu_ucode_release(&adev->mes.fw[pipe]); in amdgpu_mes_init_microcode()
1548 struct amdgpu_device *adev = m->private; in amdgpu_debugfs_mes_event_log_show()
1549 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); in amdgpu_debugfs_mes_event_log_show() local
1552 mem, PAGE_SIZE, false); in amdgpu_debugfs_mes_event_log_show()
1566 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_debugfs_mes_event_log_init()
1567 struct dentry *root = minor->debugfs_root; in amdgpu_debugfs_mes_event_log_init()