57 /* For process termination handling */
185 	mm = get_task_mm(pdd->process->lead_thread);  in kfd_sdma_activity_worker()
256  * by current process. Translates acquired wave count into number of compute units
284 	proc = pdd->process;  in kfd_get_cu_occupancy()
286 		pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",  in kfd_get_cu_occupancy()
526 	if (!q || !q->process)  in kfd_procfs_add_queue()
528 	proc = q->process;  in kfd_procfs_add_queue()
733 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
734  *	This function should be only called right after the process
787  *	process for IB usage The memory reserved is for KFD to submit
830 	struct kfd_process *process;  in kfd_create_process()  local
842 	/* If the process just called exec(3), it is possible that the  in kfd_create_process()
844 	 * of the old process image) is still in the cleanup work queue.  in kfd_create_process()
846 	 * resource for this process.  in kfd_create_process()
851 	 * take kfd processes mutex before starting of process creation  in kfd_create_process()
852 	 * so there won't be a case where two threads of the same process  in kfd_create_process()
858 		pr_debug("KFD is locked! Cannot create process");  in kfd_create_process()
859 		process = ERR_PTR(-EINVAL);  in kfd_create_process()
863 	/* A prior open of /dev/kfd could have already created the process.  in kfd_create_process()
864 	 * find_process will increase process kref in this case  in kfd_create_process()
866 	process = find_process(thread, true);  in kfd_create_process()
867 	if (process) {  in kfd_create_process()
868 		pr_debug("Process already found\n");  in kfd_create_process()
870 		process = create_process(thread);  in kfd_create_process()
871 		if (IS_ERR(process))  in kfd_create_process()
877 		process->kobj = kfd_alloc_struct(process->kobj);  in kfd_create_process()
878 		if (!process->kobj) {  in kfd_create_process()
882 		ret = kobject_init_and_add(process->kobj, &procfs_type,  in kfd_create_process()
884 					   (int)process->lead_thread->pid);  in kfd_create_process()
887 			kobject_put(process->kobj);  in kfd_create_process()
891 		kfd_sysfs_create_file(process->kobj, &process->attr_pasid,  in kfd_create_process()
894 		process->kobj_queues = kobject_create_and_add("queues",  in kfd_create_process()
895 							process->kobj);  in kfd_create_process()
896 		if (!process->kobj_queues)  in kfd_create_process()
899 		kfd_procfs_add_sysfs_stats(process);  in kfd_create_process()
900 		kfd_procfs_add_sysfs_files(process);  in kfd_create_process()
901 		kfd_procfs_add_sysfs_counters(process);  in kfd_create_process()
903 		init_waitqueue_head(&process->wait_irq_drain);  in kfd_create_process()
909 	return process;  in kfd_create_process()
914 	struct kfd_process *process;  in kfd_get_process()  local
923 	process = find_process(thread, false);  in kfd_get_process()
924 	if (!process)  in kfd_get_process()
927 	return process;  in kfd_get_process()
932 	struct kfd_process *process;  in find_process_by_mm()  local
934 	hash_for_each_possible_rcu(kfd_processes_table, process,  in find_process_by_mm()
936 		if (process->mm == mm)  in find_process_by_mm()
937 			return process;  in find_process_by_mm()
962 /* This increments the process->ref counter. */
985 	struct kfd_process *p = pdd->process;  in kfd_process_device_free_bos()
1148 /* No process locking is needed in this function, because the process
1150  * using it any more, otherwise we couldn't safely free the process
1204 	/* This increments p->ref counter if kfd process p exists */  in kfd_process_alloc_notifier()
1307 	 * Move all remaining kfd_process from the process table to a  in kfd_cleanup_processes()
1401 					pdd->process->debug_trap_enabled);  in kfd_process_device_init_cwsr_dgpu()
1450 	 * XNACK mode for this process that's compatible with all GPUs.  in kfd_process_xnack_mode()
1468 		 * per-process XNACK mode selection. But let the dev->noretry  in kfd_process_xnack_mode()
1510 	struct kfd_process *process;  in create_process()  local
1514 	process = kzalloc(sizeof(*process), GFP_KERNEL);  in create_process()
1515 	if (!process)  in create_process()
1518 	kref_init(&process->ref);  in create_process()
1519 	mutex_init(&process->mutex);  in create_process()
1520 	process->mm = thread->mm;  in create_process()
1521 	process->lead_thread = thread->group_leader;  in create_process()
1522 	process->n_pdds = 0;  in create_process()
1523 	process->queues_paused = false;  in create_process()
1524 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);  in create_process()
1525 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);  in create_process()
1526 	process->last_restore_timestamp = get_jiffies_64();  in create_process()
1527 	err = kfd_event_init_process(process);  in create_process()
1530 	process->is_32bit_user_mode = in_compat_syscall();  in create_process()
1531 	process->debug_trap_enabled = false;  in create_process()
1532 	process->debugger_process = NULL;  in create_process()
1533 	process->exception_enable_mask = 0;  in create_process()
1534 	atomic_set(&process->debugged_process_count, 0);  in create_process()
1535 	sema_init(&process->runtime_enable_sema, 0);  in create_process()
1537 	err = pqm_init(&process->pqm, process);  in create_process()
1541 	/* init process apertures*/  in create_process()
1542 	err = kfd_init_apertures(process);  in create_process()
1547 	process->xnack_enabled = kfd_process_xnack_mode(process, false);  in create_process()
1549 	err = svm_range_list_init(process);  in create_process()
1553 	/* alloc_notifier needs to find the process in the hash table */  in create_process()
1554 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,  in create_process()
1555 			(uintptr_t)process->mm);  in create_process()
1560 	kref_get(&process->ref);  in create_process()
1563 	 * because after this point we cannot unwind the process creation.  in create_process()
1565 	 * dropping the last process reference in the free_notifier.  in create_process()
1567 	mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);  in create_process()
1572 	BUG_ON(mn != &process->mmu_notifier);  in create_process()
1574 	kfd_unref_process(process);  in create_process()
1575 	get_task_struct(process->lead_thread);  in create_process()
1577 	INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);  in create_process()
1579 	return process;  in create_process()
1582 	hash_del_rcu(&process->kfd_processes);  in create_process()
1583 	svm_range_list_fini(process);  in create_process()
1585 	kfd_process_free_outstanding_kfd_bos(process);  in create_process()
1586 	kfd_process_destroy_pdds(process);  in create_process()
1588 	pqm_uninit(&process->pqm);  in create_process()
1590 	kfd_event_free_process(process);  in create_process()
1592 	mutex_destroy(&process->mutex);  in create_process()
1593 	kfree(process);  in create_process()
1628 	pdd->process = p;  in kfd_create_process_device_data()
1651  * kfd_process_device_init_vm - Initialize a VM for a process-device
1653  * @pdd: The process-device
1685 	p = pdd->process;  in kfd_process_device_init_vm()
1692 		dev_err(dev->adev->dev, "Failed to create process VM object\n");  in kfd_process_device_init_vm()
1732  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1734  * Unbinding occurs when the process dies or the device is removed.
1736  * Assumes that the process lock is held.
1746 		dev_err(dev->adev->dev, "Process device data doesn't exist\n");  in kfd_bind_process_to_device()
1775 /* Create specific handle mapped to mem from process local memory idr
1776  * Assumes that the process lock is held.
1784 /* Translate specific handle from process local memory idr
1785  * Assumes that the process lock is held.
1796 /* Remove specific handle from process local memory idr
1797  * Assumes that the process lock is held.
1826 /* This increments the process->ref counter. */
1838 		kref_get(&ret_p->process->ref);  in kfd_lookup_process_by_pasid()
1841 		return ret_p->process;  in kfd_lookup_process_by_pasid()
1852 /* This increments the process->ref counter. */
1868 /* kfd_process_evict_queues - Evict all user queues of a process
1870  * Eviction is reference-counted per process-device. This means multiple
1893 			dev_err(dev, "Failed to evict process queues\n");  in kfd_process_evict_queues()
1926 /* kfd_process_restore_queues - Restore all user queues of a process */
1941 			dev_err(dev, "Failed to restore process queues\n");  in kfd_process_restore_queues()
2000 	/* Process termination destroys this worker thread. So during the  in evict_process_worker()
2005 	pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);  in evict_process_worker()
2017 		pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);  in evict_process_worker()
2019 		pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);  in evict_process_worker()
2036 		pr_debug("Finished restoring process pid %d\n",  in restore_process_helper()
2039 		pr_err("Failed to restore queues of process pid %d\n",  in restore_process_helper()
2053 	/* Process termination destroys this worker thread. So during the  in restore_process_worker()
2057 	pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);  in restore_process_worker()
2061 	 * before KFD BOs are unreserved. If not, the process can be evicted  in restore_process_worker()
2073 		pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",  in restore_process_worker()
2090 			pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);  in kfd_suspend_all_processes()
2104 			pr_err("Restore process pid %d failed during resume\n",  in kfd_resume_all_processes()
2113 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,  in kfd_reserved_mem_mmap()  argument
2124 	pdd = kfd_get_process_device_data(dev, process);  in kfd_reserved_mem_mmap()
2133 			"Error allocating per process CWSR buffer.\n");  in kfd_reserved_mem_mmap()
2139 	/* Mapping pages to user process */  in kfd_reserved_mem_mmap()
2145 /* assumes caller holds process lock. */
2155 	pdd->process->irq_drain_is_open = true;  in kfd_process_drain_interrupts()
2175 		pdd->process->irq_drain_is_open = false;  in kfd_process_drain_interrupts()
2179 	r = wait_event_interruptible(pdd->process->wait_irq_drain,  in kfd_process_drain_interrupts()
2180 				     !READ_ONCE(pdd->process->irq_drain_is_open));  in kfd_process_drain_interrupts()
2182 		pdd->process->irq_drain_is_open = false;  in kfd_process_drain_interrupts()
2312 		seq_printf(m, "Process %d PASID %d:\n",  in kfd_debugfs_mqds_by_process()