83 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)  in kfd_process_dequeue_from_device()  argument
85 	struct kfd_node *dev = pdd->dev;  in kfd_process_dequeue_from_device()
87 	if (pdd->already_dequeued)  in kfd_process_dequeue_from_device()
93 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);  in kfd_process_dequeue_from_device()
94 	if (dev->kfd->shared_resources.enable_mes && !!pdd->proc_ctx_gpu_addr &&  in kfd_process_dequeue_from_device()
97 						 pdd->proc_ctx_gpu_addr,  in kfd_process_dequeue_from_device()
98 						 ffs(pdd->dev->xcc_mask) - 1);  in kfd_process_dequeue_from_device()
101 	pdd->already_dequeued = true;  in kfd_process_dequeue_from_device()
110 	struct kfd_process_device *pdd;  in pqm_set_gws()  local
125 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_set_gws()
126 	if (!pdd) {  in pqm_set_gws()
132 	if (gws && pdd->qpd.num_gws)  in pqm_set_gws()
135 	if (!gws && pdd->qpd.num_gws == 0)  in pqm_set_gws()
143 			ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,  in pqm_set_gws()
146 			ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,  in pqm_set_gws()
161 	pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;  in pqm_set_gws()
192 	struct kfd_process_device *pdd;  in pqm_clean_queue_resource()  local
196 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_clean_queue_resource()
197 	if (!pdd) {  in pqm_clean_queue_resource()
209 		pdd->qpd.num_gws = 0;  in pqm_clean_queue_resource()
224 			struct kfd_process_device *pdd = kfd_get_process_device_data(pqn->q->device,  in pqm_uninit()  local
226 			if (pdd) {  in pqm_uninit()
227 				kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);  in pqm_uninit()
228 				kfd_queue_release_buffers(pdd, &pqn->q->properties);  in pqm_uninit()
230 				WARN_ON(!pdd);  in pqm_uninit()
319 	struct kfd_process_device *pdd;  in pqm_create_queue()  local
338 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_create_queue()
339 	if (!pdd) {  in pqm_create_queue()
350 	if (pdd->qpd.is_debug)  in pqm_create_queue()
353 	if (pdd->qpd.queue_count >= max_queues)  in pqm_create_queue()
366 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
367 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
368 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
371 	if (!pdd->proc_ctx_cpu_ptr && dev->kfd->shared_resources.enable_mes) {  in pqm_create_queue()
375 						     &pdd->proc_ctx_bo,  in pqm_create_queue()
376 						     &pdd->proc_ctx_gpu_addr,  in pqm_create_queue()
377 						     &pdd->proc_ctx_cpu_ptr,  in pqm_create_queue()
383 		memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);  in pqm_create_queue()
407 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
428 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
457 		uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,  in pqm_create_queue()
458 								       pdd->qpd.proc_doorbells,  in pqm_create_queue()
460 								       pdd->dev->kfd->device_info.doorbell_size);  in pqm_create_queue()
486 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
487 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
488 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
495 	struct kfd_process_device *pdd;  in pqm_destroy_queue()  local
518 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_destroy_queue()
519 	if (!pdd) {  in pqm_destroy_queue()
527 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);  in pqm_destroy_queue()
532 		retval = kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);  in pqm_destroy_queue()
537 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);  in pqm_destroy_queue()
540 				pdd->pasid,  in pqm_destroy_queue()
546 		kfd_queue_release_buffers(pdd, &pqn->q->properties);  in pqm_destroy_queue()
555 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_destroy_queue()
556 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_destroy_queue()
557 		dqm->ops.unregister_process(dqm, &pdd->qpd);  in pqm_destroy_queue()
579 		struct kfd_process_device *pdd;  in pqm_update_queue_properties()  local
584 		pdd = kfd_get_process_device_data(q->device, q->process);  in pqm_update_queue_properties()
585 		if (!pdd)  in pqm_update_queue_properties()
587 		vm = drm_priv_to_vm(pdd->drm_priv);  in pqm_update_queue_properties()
734 static int get_queue_data_sizes(struct kfd_process_device *pdd,  in get_queue_data_sizes()  argument
741 	ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,  in get_queue_data_sizes()
764 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_get_queue_info()  local
766 		list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in kfd_process_get_queue_info()
774 				ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in kfd_process_get_queue_info()
813 static int criu_checkpoint_queue(struct kfd_process_device *pdd,  in criu_checkpoint_queue()  argument
823 	q_data->gpu_id = pdd->user_gpu_id;  in criu_checkpoint_queue()
850 	ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);  in criu_checkpoint_queue()
860 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,  in criu_checkpoint_queues_device()  argument
870 	list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in criu_checkpoint_queues_device()
885 		ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in criu_checkpoint_queues_device()
912 		ret = criu_checkpoint_queue(pdd, q, q_data);  in criu_checkpoint_queues_device()
940 		struct kfd_process_device *pdd = p->pdds[pdd_index];  in kfd_criu_checkpoint_queues()  local
946 		ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,  in kfd_criu_checkpoint_queues()
986 	struct kfd_process_device *pdd;  in kfd_criu_restore_queue()  local
1027 	pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);  in kfd_criu_restore_queue()
1028 	if (!pdd) {  in kfd_criu_restore_queue()
1042 	set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask));  in kfd_criu_restore_queue()
1046 	ret = pqm_create_queue(&p->pqm, pdd->dev, &qp, &queue_id, q_data, mqd, ctl_stack, NULL);  in kfd_criu_restore_queue()
1053 		ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);  in kfd_criu_restore_queue()