1029 				struct amdkfd_process_info *process_info,  in add_kgd_mem_to_kfd_bo_list()  argument
1032 	mutex_lock(&process_info->lock);  in add_kgd_mem_to_kfd_bo_list()
1035 			      &process_info->userptr_valid_list);  in add_kgd_mem_to_kfd_bo_list()
1037 		list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);  in add_kgd_mem_to_kfd_bo_list()
1038 	mutex_unlock(&process_info->lock);  in add_kgd_mem_to_kfd_bo_list()
1042 		struct amdkfd_process_info *process_info)  in remove_kgd_mem_from_kfd_bo_list()  argument
1044 	mutex_lock(&process_info->lock);  in remove_kgd_mem_from_kfd_bo_list()
1046 	mutex_unlock(&process_info->lock);  in remove_kgd_mem_from_kfd_bo_list()
1064 	struct amdkfd_process_info *process_info = mem->process_info;  in init_user_pages()  local
1070 	mutex_lock(&process_info->lock);  in init_user_pages()
1092 		mutex_lock(&process_info->notifier_lock);  in init_user_pages()
1094 		mutex_unlock(&process_info->notifier_lock);  in init_user_pages()
1095 		mutex_unlock(&process_info->lock);  in init_user_pages()
1135 	mutex_unlock(&process_info->lock);  in init_user_pages()
1351 static int process_validate_vms(struct amdkfd_process_info *process_info,  in process_validate_vms()  argument
1357 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_validate_vms()
1367 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,  in process_sync_pds_resv()  argument
1373 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_sync_pds_resv()
1387 static int process_update_pds(struct amdkfd_process_info *process_info,  in process_update_pds()  argument
1393 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in process_update_pds()
1403 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,  in init_kfd_vm()  argument
1410 	process = container_of(process_info, struct kfd_process, kgd_process_info);  in init_kfd_vm()
1411 	if (!*process_info) {  in init_kfd_vm()
1439 		*process_info = info;  in init_kfd_vm()
1442 	if (cmpxchg(&vm->process_info, NULL, *process_info) != NULL) {  in init_kfd_vm()
1464 			   &vm->process_info->eviction_fence->base,  in init_kfd_vm()
1469 	mutex_lock(&vm->process_info->lock);  in init_kfd_vm()
1471 			&(vm->process_info->vm_list_head));  in init_kfd_vm()
1472 	vm->process_info->n_vms++;  in init_kfd_vm()
1474 		*ef = dma_fence_get(&vm->process_info->eviction_fence->base);  in init_kfd_vm()
1475 	mutex_unlock(&vm->process_info->lock);  in init_kfd_vm()
1484 	vm->process_info = NULL;  in init_kfd_vm()
1488 		*process_info = NULL;  in init_kfd_vm()
1566 					   void **process_info,  in amdgpu_amdkfd_gpuvm_acquire_process_vm()  argument
1572 	if (avm->process_info)  in amdgpu_amdkfd_gpuvm_acquire_process_vm()
1581 	ret = init_kfd_vm(avm, process_info, ef);  in amdgpu_amdkfd_gpuvm_acquire_process_vm()
1593 	struct amdkfd_process_info *process_info = vm->process_info;  in amdgpu_amdkfd_gpuvm_destroy_cb()  local
1595 	if (!process_info)  in amdgpu_amdkfd_gpuvm_destroy_cb()
1599 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1600 	process_info->n_vms--;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1602 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1604 	vm->process_info = NULL;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1607 	if (!process_info->n_vms) {  in amdgpu_amdkfd_gpuvm_destroy_cb()
1608 		WARN_ON(!list_empty(&process_info->kfd_bo_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1609 		WARN_ON(!list_empty(&process_info->userptr_valid_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1610 		WARN_ON(!list_empty(&process_info->userptr_inval_list));  in amdgpu_amdkfd_gpuvm_destroy_cb()
1612 		dma_fence_put(&process_info->eviction_fence->base);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1613 		cancel_delayed_work_sync(&process_info->restore_userptr_work);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1614 		put_pid(process_info->pid);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1615 		mutex_destroy(&process_info->lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1616 		mutex_destroy(&process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1617 		kfree(process_info);  in amdgpu_amdkfd_gpuvm_destroy_cb()
1840 	(*mem)->process_info = avm->process_info;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1842 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1859 		mutex_lock(&avm->process_info->lock);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1860 		if (avm->process_info->eviction_fence &&  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1861 		    !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1863 				&avm->process_info->eviction_fence->base);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1864 		mutex_unlock(&avm->process_info->lock);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1877 	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1905 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()  local
1937 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1940 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1954 					process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
2038 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2045 		mutex_lock(&mem->process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2047 		mutex_unlock(&mem->process_info->notifier_lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2116 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2254 		bo, bo->vm_bo->vm->process_info->eviction_fence);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2295 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2316 		bo, mem->process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2323 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2331 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2413 	(*mem)->process_info = avm->process_info;  in import_obj_create()
2414 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);  in import_obj_create()
2418 	mutex_lock(&avm->process_info->lock);  in import_obj_create()
2419 	if (avm->process_info->eviction_fence &&  in import_obj_create()
2420 	    !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))  in import_obj_create()
2422 				&avm->process_info->eviction_fence->base);  in import_obj_create()
2423 	mutex_unlock(&avm->process_info->lock);  in import_obj_create()
2430 	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);  in import_obj_create()
2502 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_evict_userptr()  local
2508 	if (READ_ONCE(process_info->block_mmu_notifications))  in amdgpu_amdkfd_evict_userptr()
2511 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_evict_userptr()
2515 	if (++process_info->evicted_bos == 1) {  in amdgpu_amdkfd_evict_userptr()
2525 				&process_info->restore_userptr_work,  in amdgpu_amdkfd_evict_userptr()
2528 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_evict_userptr()
2539 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,  in update_invalid_user_pages()  argument
2548 	mutex_lock(&process_info->notifier_lock);  in update_invalid_user_pages()
2552 				 &process_info->userptr_valid_list,  in update_invalid_user_pages()
2556 				       &process_info->userptr_inval_list);  in update_invalid_user_pages()
2559 	list_for_each_entry(mem, &process_info->userptr_inval_list,  in update_invalid_user_pages()
2576 		mutex_unlock(&process_info->notifier_lock);  in update_invalid_user_pages()
2621 								pid_nr(process_info->pid), mem->va);  in update_invalid_user_pages()
2624 				p = kfd_lookup_process_by_pid(process_info->pid);  in update_invalid_user_pages()
2636 		mutex_lock(&process_info->notifier_lock);  in update_invalid_user_pages()
2651 	mutex_unlock(&process_info->notifier_lock);  in update_invalid_user_pages()
2661 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)  in validate_invalid_user_pages()  argument
2678 		list_for_each_entry(peer_vm, &process_info->vm_list_head,  in validate_invalid_user_pages()
2687 		list_for_each_entry(mem, &process_info->userptr_inval_list,  in validate_invalid_user_pages()
2699 	ret = process_validate_vms(process_info, NULL);  in validate_invalid_user_pages()
2705 				 &process_info->userptr_inval_list,  in validate_invalid_user_pages()
2736 				mutex_lock(&process_info->notifier_lock);  in validate_invalid_user_pages()
2738 				mutex_unlock(&process_info->notifier_lock);  in validate_invalid_user_pages()
2745 	ret = process_update_pds(process_info, &sync);  in validate_invalid_user_pages()
2759 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)  in confirm_valid_user_pages_locked()  argument
2765 				 &process_info->userptr_inval_list,  in confirm_valid_user_pages_locked()
2791 			       &process_info->userptr_valid_list);  in confirm_valid_user_pages_locked()
2806 	struct amdkfd_process_info *process_info =  in amdgpu_amdkfd_restore_userptr_worker()  local
2813 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2814 	evicted_bos = process_info->evicted_bos;  in amdgpu_amdkfd_restore_userptr_worker()
2815 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2820 	usertask = get_pid_task(process_info->pid, PIDTYPE_PID);  in amdgpu_amdkfd_restore_userptr_worker()
2829 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_restore_userptr_worker()
2831 	if (update_invalid_user_pages(process_info, mm))  in amdgpu_amdkfd_restore_userptr_worker()
2837 	if (!list_empty(&process_info->userptr_inval_list)) {  in amdgpu_amdkfd_restore_userptr_worker()
2838 		if (validate_invalid_user_pages(process_info))  in amdgpu_amdkfd_restore_userptr_worker()
2846 	mutex_lock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2847 	if (process_info->evicted_bos != evicted_bos)  in amdgpu_amdkfd_restore_userptr_worker()
2850 	if (confirm_valid_user_pages_locked(process_info)) {  in amdgpu_amdkfd_restore_userptr_worker()
2855 	process_info->evicted_bos = evicted_bos = 0;  in amdgpu_amdkfd_restore_userptr_worker()
2865 	mutex_unlock(&process_info->notifier_lock);  in amdgpu_amdkfd_restore_userptr_worker()
2867 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_restore_userptr_worker()
2872 			&process_info->restore_userptr_work,  in amdgpu_amdkfd_restore_userptr_worker()
2918 	struct amdkfd_process_info *process_info = info;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2930 	mutex_lock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2934 		list_for_each_entry(peer_vm, &process_info->vm_list_head,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2947 		list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2964 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3001 	ret = process_validate_vms(process_info, &exec.ticket);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3008 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3026 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3038 				pid_nr(process_info->pid));  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3046 				pid_nr(process_info->pid));  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3052 	ret = process_update_pds(process_info, &sync_obj);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3061 	ret = process_sync_pds_resv(process_info, &sync_obj);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3080 	if (dma_fence_is_signaled(&process_info->eviction_fence->base)) {  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3083 				process_info->eviction_fence->base.context,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3084 				process_info->eviction_fence->mm,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3085 				NULL, process_info->context_id);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3092 		dma_fence_put(&process_info->eviction_fence->base);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3093 		process_info->eviction_fence = new_fence;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3096 		WARN_ONCE(*ef != &process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3101 	list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3106 				   &process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3110 	list_for_each_entry(peer_vm, &process_info->vm_list_head,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3115 				   &process_info->eviction_fence->base,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3123 	mutex_unlock(&process_info->lock);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
3129 	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;  in amdgpu_amdkfd_add_gws_to_process()  local
3144 	(*mem)->process_info = process_info;  in amdgpu_amdkfd_add_gws_to_process()
3145 	add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);  in amdgpu_amdkfd_add_gws_to_process()
3150 	mutex_lock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
3170 			   &process_info->eviction_fence->base,  in amdgpu_amdkfd_add_gws_to_process()
3173 	mutex_unlock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
3181 	mutex_unlock(&(*mem)->process_info->lock);  in amdgpu_amdkfd_add_gws_to_process()
3183 	remove_kgd_mem_from_kfd_bo_list(*mem, process_info);  in amdgpu_amdkfd_add_gws_to_process()
3194 	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;  in amdgpu_amdkfd_remove_gws_from_process()  local
3201 	remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);  in amdgpu_amdkfd_remove_gws_from_process()
3210 			process_info->eviction_fence);  in amdgpu_amdkfd_remove_gws_from_process()