Lines Matching +full:gfx +full:- +full:mem
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
60 /* Ordered, single-threaded workqueue for restoring evicted
63 * their BOs and result in a live-lock situation where processes
116 pdd = workarea->pdd; in kfd_sdma_activity_worker()
119 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
120 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
127 * we loop over all SDMA queues and get their counts from user-space. in kfd_sdma_activity_worker()
133 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, in kfd_sdma_activity_worker()
139 * from the qpd->queues_list. in kfd_sdma_activity_worker()
140 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. in kfd_sdma_activity_worker()
152 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_sdma_activity_worker()
153 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && in kfd_sdma_activity_worker()
154 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) in kfd_sdma_activity_worker()
163 INIT_LIST_HEAD(&sdma_q->list); in kfd_sdma_activity_worker()
164 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; in kfd_sdma_activity_worker()
165 sdma_q->queue_id = q->properties.queue_id; in kfd_sdma_activity_worker()
166 list_add_tail(&sdma_q->list, &sdma_q_list.list); in kfd_sdma_activity_worker()
171 * qpd->queues_list. Return the past activity count as the total sdma in kfd_sdma_activity_worker()
175 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
185 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
193 ret = read_sdma_queue_counter(sdma_q->rptr, &val); in kfd_sdma_activity_worker()
196 sdma_q->queue_id); in kfd_sdma_activity_worker()
198 sdma_q->sdma_val = val; in kfd_sdma_activity_worker()
199 workarea->sdma_activity_counter += val; in kfd_sdma_activity_worker()
212 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
214 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_sdma_activity_worker()
218 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && in kfd_sdma_activity_worker()
219 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) in kfd_sdma_activity_worker()
223 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && in kfd_sdma_activity_worker()
224 (sdma_q->queue_id == q->properties.queue_id)) { in kfd_sdma_activity_worker()
225 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
236 * from qpd->queues_list during SDMA usage read. Subtract the SDMA in kfd_sdma_activity_worker()
240 workarea->sdma_activity_counter -= sdma_q->sdma_val; in kfd_sdma_activity_worker()
241 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
249 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
255 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
279 dev = pdd->dev; in kfd_get_cu_occupancy()
280 if (dev->kfd2kgd->get_cu_occupancy == NULL) in kfd_get_cu_occupancy()
281 return -EINVAL; in kfd_get_cu_occupancy()
284 proc = pdd->process; in kfd_get_cu_occupancy()
285 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
286 pr_debug("Gpu-Id: %d has no active queues for process pid %d\n", in kfd_get_cu_occupancy()
287 dev->id, (int)proc->lead_thread->pid); in kfd_get_cu_occupancy()
297 return -ENOMEM; in kfd_get_cu_occupancy()
300 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition. in kfd_get_cu_occupancy()
306 dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy, in kfd_get_cu_occupancy()
307 &max_waves_per_cu, ffs(dev->xcc_mask) - 1); in kfd_get_cu_occupancy()
311 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd, in kfd_get_cu_occupancy()
317 wave_cnt += (NUM_XCC(dev->xcc_mask) * in kfd_get_cu_occupancy()
323 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; in kfd_get_cu_occupancy()
331 if (strcmp(attr->name, "pasid") == 0) in kfd_procfs_show()
333 else if (strncmp(attr->name, "vram_", 5) == 0) { in kfd_procfs_show()
336 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage)); in kfd_procfs_show()
337 } else if (strncmp(attr->name, "sdma_", 5) == 0) { in kfd_procfs_show()
358 return -EINVAL; in kfd_procfs_show()
387 &kfd_device->kobj, "proc"); in kfd_procfs_init()
409 if (!strcmp(attr->name, "size")) in kfd_procfs_queue_show()
411 q->properties.queue_size); in kfd_procfs_queue_show()
412 else if (!strcmp(attr->name, "type")) in kfd_procfs_queue_show()
413 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); in kfd_procfs_queue_show()
414 else if (!strcmp(attr->name, "gpuid")) in kfd_procfs_queue_show()
415 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); in kfd_procfs_queue_show()
425 if (strcmp(attr->name, "evicted_ms") == 0) { in kfd_procfs_stats_show()
431 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
439 } else if (strcmp(attr->name, "cu_occupancy") == 0) { in kfd_procfs_stats_show()
453 if (!strcmp(attr->name, "faults")) { in kfd_sysfs_counters_show()
456 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
458 if (!strcmp(attr->name, "page_in")) { in kfd_sysfs_counters_show()
461 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
463 if (!strcmp(attr->name, "page_out")) { in kfd_sysfs_counters_show()
466 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
526 if (!q || !q->process) in kfd_procfs_add_queue()
527 return -EINVAL; in kfd_procfs_add_queue()
528 proc = q->process; in kfd_procfs_add_queue()
531 if (!proc->kobj_queues) in kfd_procfs_add_queue()
532 return -EFAULT; in kfd_procfs_add_queue()
533 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type, in kfd_procfs_add_queue()
534 proc->kobj_queues, "%u", q->properties.queue_id); in kfd_procfs_add_queue()
537 q->properties.queue_id); in kfd_procfs_add_queue()
538 kobject_put(&q->kobj); in kfd_procfs_add_queue()
553 attr->name = name; in kfd_sysfs_create_file()
554 attr->mode = KFD_SYSFS_FILE_MODE; in kfd_sysfs_create_file()
559 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret); in kfd_sysfs_create_file()
568 if (!p || !p->kobj) in kfd_procfs_add_sysfs_stats()
573 * - proc/<pid>/stats_<gpuid>/ in kfd_procfs_add_sysfs_stats()
574 * - proc/<pid>/stats_<gpuid>/evicted_ms in kfd_procfs_add_sysfs_stats()
575 * - proc/<pid>/stats_<gpuid>/cu_occupancy in kfd_procfs_add_sysfs_stats()
577 for (i = 0; i < p->n_pdds; i++) { in kfd_procfs_add_sysfs_stats()
578 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats()
581 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
582 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
583 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
586 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
588 p->kobj, in kfd_procfs_add_sysfs_stats()
594 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
595 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
599 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
602 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
603 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
604 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
615 if (!p || !p->kobj) in kfd_procfs_add_sysfs_counters()
620 * - proc/<pid>/counters_<gpuid>/ in kfd_procfs_add_sysfs_counters()
621 * - proc/<pid>/counters_<gpuid>/faults in kfd_procfs_add_sysfs_counters()
622 * - proc/<pid>/counters_<gpuid>/page_in in kfd_procfs_add_sysfs_counters()
623 * - proc/<pid>/counters_<gpuid>/page_out in kfd_procfs_add_sysfs_counters()
625 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) { in kfd_procfs_add_sysfs_counters()
626 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters()
630 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
636 p->kobj, counters_dir_filename); in kfd_procfs_add_sysfs_counters()
644 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
645 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
647 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
649 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
658 if (!p || !p->kobj) in kfd_procfs_add_sysfs_files()
663 * - proc/<pid>/vram_<gpuid> in kfd_procfs_add_sysfs_files()
664 * - proc/<pid>/sdma_<gpuid> in kfd_procfs_add_sysfs_files()
666 for (i = 0; i < p->n_pdds; i++) { in kfd_procfs_add_sysfs_files()
667 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files()
669 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
670 pdd->dev->id); in kfd_procfs_add_sysfs_files()
671 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
672 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
674 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
675 pdd->dev->id); in kfd_procfs_add_sysfs_files()
676 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
677 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
686 kobject_del(&q->kobj); in kfd_procfs_del_queue()
687 kobject_put(&q->kobj); in kfd_procfs_del_queue()
700 return -ENOMEM; in kfd_process_create_wq()
718 static void kfd_process_free_gpuvm(struct kgd_mem *mem, in kfd_process_free_gpuvm() argument
721 struct kfd_node *dev = pdd->dev; in kfd_process_free_gpuvm()
724 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); in kfd_process_free_gpuvm()
728 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
729 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
733 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
737 * not need to take p->mutex.
741 uint32_t flags, struct kgd_mem **mem, void **kptr) in kfd_process_alloc_gpuvm() argument
743 struct kfd_node *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
746 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size, in kfd_process_alloc_gpuvm()
747 pdd->drm_priv, mem, NULL, in kfd_process_alloc_gpuvm()
752 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem, in kfd_process_alloc_gpuvm()
753 pdd->drm_priv); in kfd_process_alloc_gpuvm()
757 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true); in kfd_process_alloc_gpuvm()
765 (struct kgd_mem *)*mem, kptr, NULL); in kfd_process_alloc_gpuvm()
775 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv); in kfd_process_alloc_gpuvm()
778 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
781 *mem = NULL; in kfd_process_alloc_gpuvm()
786 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
794 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
799 struct kgd_mem *mem; in kfd_process_device_reserve_ib_mem() local
803 if (qpd->ib_kaddr || !qpd->ib_base) in kfd_process_device_reserve_ib_mem()
807 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
808 &mem, &kaddr); in kfd_process_device_reserve_ib_mem()
812 qpd->ib_mem = mem; in kfd_process_device_reserve_ib_mem()
813 qpd->ib_kaddr = kaddr; in kfd_process_device_reserve_ib_mem()
820 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_ib_mem()
822 if (!qpd->ib_kaddr || !qpd->ib_base) in kfd_process_device_destroy_ib_mem()
825 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr); in kfd_process_device_destroy_ib_mem()
833 if (!(thread->mm && mmget_not_zero(thread->mm))) in kfd_create_process()
834 return ERR_PTR(-EINVAL); in kfd_create_process()
837 if (thread->group_leader->mm != thread->mm) { in kfd_create_process()
838 mmput(thread->mm); in kfd_create_process()
839 return ERR_PTR(-EINVAL); in kfd_create_process()
859 process = ERR_PTR(-EINVAL); in kfd_create_process()
877 process->kobj = kfd_alloc_struct(process->kobj); in kfd_create_process()
878 if (!process->kobj) { in kfd_create_process()
882 ret = kobject_init_and_add(process->kobj, &procfs_type, in kfd_create_process()
884 (int)process->lead_thread->pid); in kfd_create_process()
887 kobject_put(process->kobj); in kfd_create_process()
891 kfd_sysfs_create_file(process->kobj, &process->attr_pasid, in kfd_create_process()
894 process->kobj_queues = kobject_create_and_add("queues", in kfd_create_process()
895 process->kobj); in kfd_create_process()
896 if (!process->kobj_queues) in kfd_create_process()
903 init_waitqueue_head(&process->wait_irq_drain); in kfd_create_process()
907 mmput(thread->mm); in kfd_create_process()
916 if (!thread->mm) in kfd_get_process()
917 return ERR_PTR(-EINVAL); in kfd_get_process()
920 if (thread->group_leader->mm != thread->mm) in kfd_get_process()
921 return ERR_PTR(-EINVAL); in kfd_get_process()
925 return ERR_PTR(-EINVAL); in kfd_get_process()
936 if (process->mm == mm) in find_process_by_mm()
949 p = find_process_by_mm(thread->mm); in find_process()
951 kref_get(&p->ref); in find_process()
959 kref_put(&p->ref, kfd_process_ref_release); in kfd_unref_process()
962 /* This increments the process->ref counter. */
985 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
986 void *mem; in kfd_process_device_free_bos() local
994 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
996 for (i = 0; i < p->n_pdds; i++) { in kfd_process_device_free_bos()
997 struct kfd_process_device *peer_pdd = p->pdds[i]; in kfd_process_device_free_bos()
999 if (!peer_pdd->drm_priv) in kfd_process_device_free_bos()
1002 peer_pdd->dev->adev, mem, peer_pdd->drm_priv); in kfd_process_device_free_bos()
1005 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem, in kfd_process_device_free_bos()
1006 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
1019 void *mem; in kfd_process_kunmap_signal_bo() local
1021 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle)); in kfd_process_kunmap_signal_bo()
1025 mutex_lock(&p->mutex); in kfd_process_kunmap_signal_bo()
1031 mem = kfd_process_device_translate_handle( in kfd_process_kunmap_signal_bo()
1032 pdd, GET_IDR_HANDLE(p->signal_handle)); in kfd_process_kunmap_signal_bo()
1033 if (!mem) in kfd_process_kunmap_signal_bo()
1036 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); in kfd_process_kunmap_signal_bo()
1039 mutex_unlock(&p->mutex); in kfd_process_kunmap_signal_bo()
1046 for (i = 0; i < p->n_pdds; i++) in kfd_process_free_outstanding_kfd_bos()
1047 kfd_process_device_free_bos(p->pdds[i]); in kfd_process_free_outstanding_kfd_bos()
1054 for (i = 0; i < p->n_pdds; i++) { in kfd_process_destroy_pdds()
1055 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds()
1058 pdd->dev->id, p->lead_thread->pid); in kfd_process_destroy_pdds()
1062 if (pdd->drm_file) in kfd_process_destroy_pdds()
1063 fput(pdd->drm_file); in kfd_process_destroy_pdds()
1065 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
1066 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
1069 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
1071 kfd_free_process_doorbells(pdd->dev->kfd, pdd); in kfd_process_destroy_pdds()
1073 if (pdd->dev->kfd->shared_resources.enable_mes && in kfd_process_destroy_pdds()
1074 pdd->proc_ctx_cpu_ptr) in kfd_process_destroy_pdds()
1075 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev, in kfd_process_destroy_pdds()
1076 &pdd->proc_ctx_bo); in kfd_process_destroy_pdds()
1081 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
1082 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1083 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1084 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
1088 p->pdds[i] = NULL; in kfd_process_destroy_pdds()
1090 p->n_pdds = 0; in kfd_process_destroy_pdds()
1098 if (!p->kobj) in kfd_process_remove_sysfs()
1101 sysfs_remove_file(p->kobj, &p->attr_pasid); in kfd_process_remove_sysfs()
1102 kobject_del(p->kobj_queues); in kfd_process_remove_sysfs()
1103 kobject_put(p->kobj_queues); in kfd_process_remove_sysfs()
1104 p->kobj_queues = NULL; in kfd_process_remove_sysfs()
1106 for (i = 0; i < p->n_pdds; i++) { in kfd_process_remove_sysfs()
1107 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1109 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1110 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1112 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1113 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1114 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1115 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1116 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1117 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1118 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1121 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) { in kfd_process_remove_sysfs()
1122 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1124 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1125 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1126 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1127 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1128 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1129 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1132 kobject_del(p->kobj); in kfd_process_remove_sysfs()
1133 kobject_put(p->kobj); in kfd_process_remove_sysfs()
1134 p->kobj = NULL; in kfd_process_remove_sysfs()
1144 for (i = 0; i < p->n_pdds; i++) in kfd_process_wait_gpu_reset_complete()
1145 flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq); in kfd_process_wait_gpu_reset_complete()
1160 pqm_uninit(&p->pqm); in kfd_process_wq_release()
1172 ef = rcu_access_pointer(p->ef); in kfd_process_wq_release()
1187 mutex_destroy(&p->mutex); in kfd_process_wq_release()
1189 put_task_struct(p->lead_thread); in kfd_process_wq_release()
1198 INIT_WORK(&p->release_work, kfd_process_wq_release); in kfd_process_ref_release()
1199 queue_work(kfd_process_wq, &p->release_work); in kfd_process_ref_release()
1204 /* This increments p->ref counter if kfd process p exists */ in kfd_process_alloc_notifier()
1207 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH); in kfd_process_alloc_notifier()
1219 cancel_delayed_work_sync(&p->eviction_work); in kfd_process_notifier_release_internal()
1220 cancel_delayed_work_sync(&p->restore_work); in kfd_process_notifier_release_internal()
1222 for (i = 0; i < p->n_pdds; i++) { in kfd_process_notifier_release_internal()
1223 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_notifier_release_internal()
1225 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */ in kfd_process_notifier_release_internal()
1226 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup) in kfd_process_notifier_release_internal()
1227 amdgpu_gfx_off_ctrl(pdd->dev->adev, true); in kfd_process_notifier_release_internal()
1231 p->mm = NULL; in kfd_process_notifier_release_internal()
1234 if (atomic_read(&p->debugged_process_count) > 0) { in kfd_process_notifier_release_internal()
1240 if (target->debugger_process && target->debugger_process == p) { in kfd_process_notifier_release_internal()
1241 mutex_lock_nested(&target->mutex, 1); in kfd_process_notifier_release_internal()
1243 mutex_unlock(&target->mutex); in kfd_process_notifier_release_internal()
1244 if (atomic_read(&p->debugged_process_count) == 0) in kfd_process_notifier_release_internal()
1252 mmu_notifier_put(&p->mmu_notifier); in kfd_process_notifier_release_internal()
1265 if (WARN_ON(p->mm != mm)) in kfd_process_notifier_release()
1280 hash_del_rcu(&p->kfd_processes); in kfd_process_notifier_release()
1314 hash_del_rcu(&p->kfd_processes); in kfd_cleanup_processes()
1316 hlist_add_head(&p->kfd_processes, &cleanup_list); in kfd_cleanup_processes()
1335 if (p->has_cwsr) in kfd_process_init_cwsr_apu()
1338 for (i = 0; i < p->n_pdds; i++) { in kfd_process_init_cwsr_apu()
1339 struct kfd_node *dev = p->pdds[i]->dev; in kfd_process_init_cwsr_apu()
1340 struct qcm_process_device *qpd = &p->pdds[i]->qpd; in kfd_process_init_cwsr_apu()
1342 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) in kfd_process_init_cwsr_apu()
1345 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id); in kfd_process_init_cwsr_apu()
1346 qpd->tba_addr = (int64_t)vm_mmap(filep, 0, in kfd_process_init_cwsr_apu()
1350 if (IS_ERR_VALUE(qpd->tba_addr)) { in kfd_process_init_cwsr_apu()
1351 int err = qpd->tba_addr; in kfd_process_init_cwsr_apu()
1353 dev_err(dev->adev->dev, in kfd_process_init_cwsr_apu()
1355 qpd->tba_addr = 0; in kfd_process_init_cwsr_apu()
1356 qpd->cwsr_kaddr = NULL; in kfd_process_init_cwsr_apu()
1360 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size); in kfd_process_init_cwsr_apu()
1362 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled); in kfd_process_init_cwsr_apu()
1364 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; in kfd_process_init_cwsr_apu()
1366 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); in kfd_process_init_cwsr_apu()
1369 p->has_cwsr = true; in kfd_process_init_cwsr_apu()
1376 struct kfd_node *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1377 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1381 struct kgd_mem *mem; in kfd_process_device_init_cwsr_dgpu() local
1385 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) in kfd_process_device_init_cwsr_dgpu()
1389 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1390 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr); in kfd_process_device_init_cwsr_dgpu()
1394 qpd->cwsr_mem = mem; in kfd_process_device_init_cwsr_dgpu()
1395 qpd->cwsr_kaddr = kaddr; in kfd_process_device_init_cwsr_dgpu()
1396 qpd->tba_addr = qpd->cwsr_base; in kfd_process_device_init_cwsr_dgpu()
1398 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size); in kfd_process_device_init_cwsr_dgpu()
1400 kfd_process_set_trap_debug_flag(&pdd->qpd, in kfd_process_device_init_cwsr_dgpu()
1401 pdd->process->debug_trap_enabled); in kfd_process_device_init_cwsr_dgpu()
1403 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; in kfd_process_device_init_cwsr_dgpu()
1405 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); in kfd_process_device_init_cwsr_dgpu()
1412 struct kfd_node *dev = pdd->dev; in kfd_process_device_destroy_cwsr_dgpu()
1413 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_cwsr_dgpu()
1415 if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base) in kfd_process_device_destroy_cwsr_dgpu()
1418 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr); in kfd_process_device_destroy_cwsr_dgpu()
1425 if (qpd->cwsr_kaddr) { in kfd_process_set_trap_handler()
1426 /* KFD trap handler is bound, record as second-level TBA/TMA in kfd_process_set_trap_handler()
1427 * in first-level TMA. First-level trap will jump to second. in kfd_process_set_trap_handler()
1430 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET); in kfd_process_set_trap_handler()
1434 /* No trap handler bound, bind as first-level TBA/TMA. */ in kfd_process_set_trap_handler()
1435 qpd->tba_addr = tba_addr; in kfd_process_set_trap_handler()
1436 qpd->tma_addr = tma_addr; in kfd_process_set_trap_handler()
1452 * built for XNACK-off. On GFXv9 it may perform slower. in kfd_process_xnack_mode()
1454 * Therefore applications built for XNACK-off can always be in kfd_process_xnack_mode()
1458 for (i = 0; i < p->n_pdds; i++) { in kfd_process_xnack_mode()
1459 struct kfd_node *dev = p->pdds[i]->dev; in kfd_process_xnack_mode()
1468 * per-process XNACK mode selection. But let the dev->noretry in kfd_process_xnack_mode()
1472 if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) { in kfd_process_xnack_mode()
1481 * management and memory-manager-related preemptions or in kfd_process_xnack_mode()
1487 if (dev->kfd->noretry) in kfd_process_xnack_mode()
1497 if (qpd->cwsr_kaddr) { in kfd_process_set_trap_debug_flag()
1499 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET); in kfd_process_set_trap_debug_flag()
1512 int err = -ENOMEM; in create_process()
1518 kref_init(&process->ref); in create_process()
1519 mutex_init(&process->mutex); in create_process()
1520 process->mm = thread->mm; in create_process()
1521 process->lead_thread = thread->group_leader; in create_process()
1522 process->n_pdds = 0; in create_process()
1523 process->queues_paused = false; in create_process()
1524 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); in create_process()
1525 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); in create_process()
1526 process->last_restore_timestamp = get_jiffies_64(); in create_process()
1530 process->is_32bit_user_mode = in_compat_syscall(); in create_process()
1531 process->debug_trap_enabled = false; in create_process()
1532 process->debugger_process = NULL; in create_process()
1533 process->exception_enable_mask = 0; in create_process()
1534 atomic_set(&process->debugged_process_count, 0); in create_process()
1535 sema_init(&process->runtime_enable_sema, 0); in create_process()
1537 err = pqm_init(&process->pqm, process); in create_process()
1547 process->xnack_enabled = kfd_process_xnack_mode(process, false); in create_process()
1554 hash_add_rcu(kfd_processes_table, &process->kfd_processes, in create_process()
1555 (uintptr_t)process->mm); in create_process()
1560 kref_get(&process->ref); in create_process()
1567 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm); in create_process()
1572 BUG_ON(mn != &process->mmu_notifier); in create_process()
1575 get_task_struct(process->lead_thread); in create_process()
1577 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler); in create_process()
1582 hash_del_rcu(&process->kfd_processes); in create_process()
1588 pqm_uninit(&process->pqm); in create_process()
1592 mutex_destroy(&process->mutex); in create_process()
1603 for (i = 0; i < p->n_pdds; i++) in kfd_get_process_device_data()
1604 if (p->pdds[i]->dev == dev) in kfd_get_process_device_data()
1605 return p->pdds[i]; in kfd_get_process_device_data()
1615 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE)) in kfd_create_process_device_data()
1621 pdd->dev = dev; in kfd_create_process_device_data()
1622 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1623 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1624 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1625 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1626 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1627 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1628 pdd->process = p; in kfd_create_process_device_data()
1629 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1630 pdd->already_dequeued = false; in kfd_create_process_device_data()
1631 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1632 atomic64_set(&pdd->vram_usage, 0); in kfd_create_process_device_data()
1633 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1634 pdd->user_gpu_id = dev->id; in kfd_create_process_device_data()
1635 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1637 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1638 if (kfd_dbg_is_per_vmid_supported(pdd->dev)) in kfd_create_process_device_data()
1639 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap( in kfd_create_process_device_data()
1640 pdd->dev->adev, in kfd_create_process_device_data()
1645 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1651 * kfd_process_device_init_vm - Initialize a VM for a process-device
1653 * @pdd: The process-device
1662 * Returns 0 on success, -errno on failure.
1675 return -EINVAL; in kfd_process_device_init_vm()
1677 if (pdd->drm_priv) in kfd_process_device_init_vm()
1678 return -EBUSY; in kfd_process_device_init_vm()
1683 avm = &drv_priv->vm; in kfd_process_device_init_vm()
1685 p = pdd->process; in kfd_process_device_init_vm()
1686 dev = pdd->dev; in kfd_process_device_init_vm()
1688 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm, in kfd_process_device_init_vm()
1689 &p->kgd_process_info, in kfd_process_device_init_vm()
1690 p->ef ? NULL : &ef); in kfd_process_device_init_vm()
1692 dev_err(dev->adev->dev, "Failed to create process VM object\n"); in kfd_process_device_init_vm()
1696 if (!p->ef) in kfd_process_device_init_vm()
1697 RCU_INIT_POINTER(p->ef, ef); in kfd_process_device_init_vm()
1699 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1708 if (unlikely(!avm->pasid)) { in kfd_process_device_init_vm()
1709 dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated", in kfd_process_device_init_vm()
1711 ret = -EINVAL; in kfd_process_device_init_vm()
1715 pdd->pasid = avm->pasid; in kfd_process_device_init_vm()
1716 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1725 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1726 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm); in kfd_process_device_init_vm()
1732 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1746 dev_err(dev->adev->dev, "Process device data doesn't exist\n"); in kfd_bind_process_to_device()
1747 return ERR_PTR(-ENOMEM); in kfd_bind_process_to_device()
1750 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1751 return ERR_PTR(-ENODEV); in kfd_bind_process_to_device()
1754 * signal runtime-pm system to auto resume and prevent in kfd_bind_process_to_device()
1758 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1759 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev); in kfd_bind_process_to_device()
1761 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev); in kfd_bind_process_to_device()
1770 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1775 /* Create specific handle mapped to mem from process local memory idr
1779 void *mem) in kfd_process_device_create_obj_handle() argument
1781 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1793 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1803 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1814 for (i = 0; i < p->n_pdds; i++) { in kfd_lookup_process_device_by_pasid()
1815 if (p->pdds[i]->pasid == pasid) { in kfd_lookup_process_device_by_pasid()
1816 ret_p = p->pdds[i]; in kfd_lookup_process_device_by_pasid()
1826 /* This increments the process->ref counter. */
1838 kref_get(&ret_p->process->ref); in kfd_lookup_process_by_pasid()
1841 return ret_p->process; in kfd_lookup_process_by_pasid()
1852 /* This increments the process->ref counter. */
1861 kref_get(&p->ref); in kfd_lookup_process_by_mm()
1868 /* kfd_process_evict_queues - Evict all user queues of a process
1870 * Eviction is reference-counted per process-device. This means multiple
1879 for (i = 0; i < p->n_pdds; i++) { in kfd_process_evict_queues()
1880 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues()
1881 struct device *dev = pdd->dev->adev->dev; in kfd_process_evict_queues()
1883 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid, in kfd_process_evict_queues()
1886 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1887 &pdd->qpd); in kfd_process_evict_queues()
1888 /* evict return -EIO if HWS is hang or asic is resetting, in this case in kfd_process_evict_queues()
1892 if (r && r != -EIO) { in kfd_process_evict_queues()
1898 pdd->dev->dqm->is_hws_hang = false; in kfd_process_evict_queues()
1907 for (i = 0; i < p->n_pdds; i++) { in kfd_process_evict_queues()
1908 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues()
1913 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_evict_queues()
1915 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1916 &pdd->qpd)) in kfd_process_evict_queues()
1917 dev_err(pdd->dev->adev->dev, in kfd_process_evict_queues()
1920 n_evicted--; in kfd_process_evict_queues()
1926 /* kfd_process_restore_queues - Restore all user queues of a process */
1932 for (i = 0; i < p->n_pdds; i++) { in kfd_process_restore_queues()
1933 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues()
1934 struct device *dev = pdd->dev->adev->dev; in kfd_process_restore_queues()
1936 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_restore_queues()
1938 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1939 &pdd->qpd); in kfd_process_restore_queues()
1954 for (i = 0; i < p->n_pdds; i++) in kfd_process_gpuidx_from_gpuid()
1955 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id) in kfd_process_gpuidx_from_gpuid()
1957 return -EINVAL; in kfd_process_gpuidx_from_gpuid()
1966 for (i = 0; i < p->n_pdds; i++) in kfd_process_gpuid_from_node()
1967 if (p->pdds[i] && p->pdds[i]->dev == node) { in kfd_process_gpuid_from_node()
1968 *gpuid = p->pdds[i]->user_gpu_id; in kfd_process_gpuid_from_node()
1972 return -EINVAL; in kfd_process_gpuid_from_node()
1981 ef = dma_fence_get_rcu_safe(&p->ef); in signal_eviction_fence()
1984 return -EINVAL; in signal_eviction_fence()
2005 pr_debug("Started evicting process pid %d\n", p->lead_thread->pid); in evict_process_worker()
2013 mod_delayed_work(kfd_restore_wq, &p->restore_work, in evict_process_worker()
2017 pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid); in evict_process_worker()
2019 pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid); in evict_process_worker()
2027 if (p->kgd_process_info) { in restore_process_helper()
2029 p->kgd_process_info, &p->ef); in restore_process_helper()
2037 p->lead_thread->pid); in restore_process_helper()
2040 p->lead_thread->pid); in restore_process_helper()
2057 pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid); in restore_process_worker()
2065 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two in restore_process_worker()
2069 p->last_restore_timestamp = get_jiffies_64(); in restore_process_worker()
2074 p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS); in restore_process_worker()
2075 if (mod_delayed_work(kfd_restore_wq, &p->restore_work, in restore_process_worker()
2090 pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid); in kfd_suspend_all_processes()
2105 p->lead_thread->pid); in kfd_resume_all_processes()
2106 ret = -EFAULT; in kfd_resume_all_processes()
2119 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) { in kfd_reserved_mem_mmap()
2120 dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n"); in kfd_reserved_mem_mmap()
2121 return -EINVAL; in kfd_reserved_mem_mmap()
2126 return -EINVAL; in kfd_reserved_mem_mmap()
2127 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
2129 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in kfd_reserved_mem_mmap()
2131 if (!qpd->cwsr_kaddr) { in kfd_reserved_mem_mmap()
2132 dev_err(dev->adev->dev, in kfd_reserved_mem_mmap()
2134 return -ENOMEM; in kfd_reserved_mem_mmap()
2140 return remap_pfn_range(vma, vma->vm_start, in kfd_reserved_mem_mmap()
2141 PFN_DOWN(__pa(qpd->cwsr_kaddr)), in kfd_reserved_mem_mmap()
2142 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot); in kfd_reserved_mem_mmap()
2152 if (!KFD_IS_SOC15(pdd->dev)) in kfd_process_drain_interrupts()
2155 pdd->process->irq_drain_is_open = true; in kfd_process_drain_interrupts()
2160 irq_drain_fence[3] = pdd->pasid; in kfd_process_drain_interrupts()
2163 * For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3] in kfd_process_drain_interrupts()
2165 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) || in kfd_process_drain_interrupts()
2166 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) || in kfd_process_drain_interrupts()
2167 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) { in kfd_process_drain_interrupts()
2168 node_id = ffs(pdd->dev->interrupt_bitmap) - 1; in kfd_process_drain_interrupts()
2173 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev, in kfd_process_drain_interrupts()
2175 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2179 r = wait_event_interruptible(pdd->process->wait_irq_drain, in kfd_process_drain_interrupts()
2180 !READ_ONCE(pdd->process->irq_drain_is_open)); in kfd_process_drain_interrupts()
2182 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2196 WRITE_ONCE(p->irq_drain_is_open, false); in kfd_process_close_interrupt_drain()
2197 wake_up_all(&p->wait_irq_drain); in kfd_process_close_interrupt_drain()
2222 p = workarea->p; in send_exception_work_handler()
2224 mm = get_task_mm(p->lead_thread); in send_exception_work_handler()
2231 q = pqm_get_user_queue(&p->pqm, workarea->queue_id); in send_exception_work_handler()
2236 csa_header = (void __user *)q->properties.ctx_save_restore_area_address; in send_exception_work_handler()
2238 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr); in send_exception_work_handler()
2240 cur_err |= workarea->error_reason; in send_exception_work_handler()
2242 get_user(ev_id, &csa_header->err_event_id); in send_exception_work_handler()
2275 for (i = 0; i < p->n_pdds; i++) { in kfd_process_device_data_by_id()
2276 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_device_data_by_id()
2278 if (pdd->user_gpu_id == gpu_id) in kfd_process_device_data_by_id()
2292 for (i = 0; i < p->n_pdds; i++) { in kfd_process_get_user_gpu_id()
2293 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_get_user_gpu_id()
2295 if (pdd->dev->id == actual_gpu_id) in kfd_process_get_user_gpu_id()
2296 return pdd->user_gpu_id; in kfd_process_get_user_gpu_id()
2298 return -EINVAL; in kfd_process_get_user_gpu_id()
2313 p->lead_thread->tgid, p->lead_thread->pid); in kfd_debugfs_mqds_by_process()
2315 mutex_lock(&p->mutex); in kfd_debugfs_mqds_by_process()
2316 r = pqm_debugfs_mqds(m, &p->pqm); in kfd_debugfs_mqds_by_process()
2317 mutex_unlock(&p->mutex); in kfd_debugfs_mqds_by_process()