1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/mutex.h>
25 #include <linux/log2.h>
26 #include <linux/sched.h>
27 #include <linux/sched/mm.h>
28 #include <linux/sched/task.h>
29 #include <linux/mmu_context.h>
30 #include <linux/slab.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 #include "amdgpu_reset.h"
39 
40 struct mm_struct;
41 
42 #include "kfd_priv.h"
43 #include "kfd_device_queue_manager.h"
44 #include "kfd_svm.h"
45 #include "kfd_smi_events.h"
46 #include "kfd_debug.h"
47 
48 /*
49  * List of struct kfd_process (field kfd_process).
50  * Unique/indexed by mm_struct*
51  */
52 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
53 DEFINE_MUTEX(kfd_processes_mutex);
54 
55 DEFINE_SRCU(kfd_processes_srcu);
56 
57 /* For process termination handling */
58 static struct workqueue_struct *kfd_process_wq;
59 
60 /* Ordered, single-threaded workqueue for restoring evicted
61  * processes. Restoring multiple processes concurrently under memory
62  * pressure can lead to processes blocking each other from validating
63  * their BOs and result in a live-lock situation where processes
64  * remain evicted indefinitely.
65  */
66 static struct workqueue_struct *kfd_restore_wq;
67 
68 static struct kfd_process *find_process(const struct task_struct *thread,
69 					bool ref);
70 static void kfd_process_ref_release(struct kref *ref);
71 static struct kfd_process *create_process(const struct task_struct *thread);
72 
73 static void evict_process_worker(struct work_struct *work);
74 static void restore_process_worker(struct work_struct *work);
75 
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77 
78 struct kfd_procfs_tree {
79 	struct kobject *kobj;
80 };
81 
82 static struct kfd_procfs_tree procfs;
83 
84 /*
85  * Structure for SDMA activity tracking
86  */
87 struct kfd_sdma_activity_handler_workarea {
88 	struct work_struct sdma_activity_work;
89 	struct kfd_process_device *pdd;
90 	uint64_t sdma_activity_counter;
91 };
92 
93 struct temp_sdma_queue_list {
94 	uint64_t __user *rptr;
95 	uint64_t sdma_val;
96 	unsigned int queue_id;
97 	struct list_head list;
98 };
99 
kfd_sdma_activity_worker(struct work_struct * work)100 static void kfd_sdma_activity_worker(struct work_struct *work)
101 {
102 	struct kfd_sdma_activity_handler_workarea *workarea;
103 	struct kfd_process_device *pdd;
104 	uint64_t val;
105 	struct mm_struct *mm;
106 	struct queue *q;
107 	struct qcm_process_device *qpd;
108 	struct device_queue_manager *dqm;
109 	int ret = 0;
110 	struct temp_sdma_queue_list sdma_q_list;
111 	struct temp_sdma_queue_list *sdma_q, *next;
112 
113 	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 				sdma_activity_work);
115 
116 	pdd = workarea->pdd;
117 	if (!pdd)
118 		return;
119 	dqm = pdd->dev->dqm;
120 	qpd = &pdd->qpd;
121 	if (!dqm || !qpd)
122 		return;
123 	/*
124 	 * Total SDMA activity is current SDMA activity + past SDMA activity
125 	 * Past SDMA count is stored in pdd.
126 	 * To get the current activity counters for all active SDMA queues,
127 	 * we loop over all SDMA queues and get their counts from user-space.
128 	 *
129 	 * We cannot call get_user() with dqm_lock held as it can cause
130 	 * a circular lock dependency situation. To read the SDMA stats,
131 	 * we need to do the following:
132 	 *
133 	 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 	 *    with dqm_lock/dqm_unlock().
135 	 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 	 *    Save the SDMA count for each node and also add the count to the total
137 	 *    SDMA count counter.
138 	 *    Its possible, during this step, a few SDMA queue nodes got deleted
139 	 *    from the qpd->queues_list.
140 	 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 	 *    If any node got deleted, its SDMA count would be captured in the sdma
142 	 *    past activity counter. So subtract the SDMA counter stored in step 2
143 	 *    for this node from the total SDMA count.
144 	 */
145 	INIT_LIST_HEAD(&sdma_q_list.list);
146 
147 	/*
148 	 * Create the temp list of all SDMA queues
149 	 */
150 	dqm_lock(dqm);
151 
152 	list_for_each_entry(q, &qpd->queues_list, list) {
153 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 			continue;
156 
157 		sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
158 		if (!sdma_q) {
159 			dqm_unlock(dqm);
160 			goto cleanup;
161 		}
162 
163 		INIT_LIST_HEAD(&sdma_q->list);
164 		sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
165 		sdma_q->queue_id = q->properties.queue_id;
166 		list_add_tail(&sdma_q->list, &sdma_q_list.list);
167 	}
168 
169 	/*
170 	 * If the temp list is empty, then no SDMA queues nodes were found in
171 	 * qpd->queues_list. Return the past activity count as the total sdma
172 	 * count
173 	 */
174 	if (list_empty(&sdma_q_list.list)) {
175 		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
176 		dqm_unlock(dqm);
177 		return;
178 	}
179 
180 	dqm_unlock(dqm);
181 
182 	/*
183 	 * Get the usage count for each SDMA queue in temp_list.
184 	 */
185 	mm = get_task_mm(pdd->process->lead_thread);
186 	if (!mm)
187 		goto cleanup;
188 
189 	kthread_use_mm(mm);
190 
191 	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
192 		val = 0;
193 		ret = read_sdma_queue_counter(sdma_q->rptr, &val);
194 		if (ret) {
195 			pr_debug("Failed to read SDMA queue active counter for queue id: %d",
196 				 sdma_q->queue_id);
197 		} else {
198 			sdma_q->sdma_val = val;
199 			workarea->sdma_activity_counter += val;
200 		}
201 	}
202 
203 	kthread_unuse_mm(mm);
204 	mmput(mm);
205 
206 	/*
207 	 * Do a second iteration over qpd_queues_list to check if any SDMA
208 	 * nodes got deleted while fetching SDMA counter.
209 	 */
210 	dqm_lock(dqm);
211 
212 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
213 
214 	list_for_each_entry(q, &qpd->queues_list, list) {
215 		if (list_empty(&sdma_q_list.list))
216 			break;
217 
218 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
219 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
220 			continue;
221 
222 		list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
223 			if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
224 			     (sdma_q->queue_id == q->properties.queue_id)) {
225 				list_del(&sdma_q->list);
226 				kfree(sdma_q);
227 				break;
228 			}
229 		}
230 	}
231 
232 	dqm_unlock(dqm);
233 
234 	/*
235 	 * If temp list is not empty, it implies some queues got deleted
236 	 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
237 	 * count for each node from the total SDMA count.
238 	 */
239 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
240 		workarea->sdma_activity_counter -= sdma_q->sdma_val;
241 		list_del(&sdma_q->list);
242 		kfree(sdma_q);
243 	}
244 
245 	return;
246 
247 cleanup:
248 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
249 		list_del(&sdma_q->list);
250 		kfree(sdma_q);
251 	}
252 }
253 
254 /**
255  * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
256  * by current process. Translates acquired wave count into number of compute units
257  * that are occupied.
258  *
259  * @attr: Handle of attribute that allows reporting of wave count. The attribute
260  * handle encapsulates GPU device it is associated with, thereby allowing collection
261  * of waves in flight, etc
262  * @buffer: Handle of user provided buffer updated with wave count
263  *
264  * Return: Number of bytes written to user buffer or an error value
265  */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)266 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
267 {
268 	int cu_cnt;
269 	int wave_cnt;
270 	int max_waves_per_cu;
271 	struct kfd_node *dev = NULL;
272 	struct kfd_process *proc = NULL;
273 	struct kfd_process_device *pdd = NULL;
274 	int i;
275 	struct kfd_cu_occupancy *cu_occupancy;
276 	u32 queue_format;
277 
278 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
279 	dev = pdd->dev;
280 	if (dev->kfd2kgd->get_cu_occupancy == NULL)
281 		return -EINVAL;
282 
283 	cu_cnt = 0;
284 	proc = pdd->process;
285 	if (pdd->qpd.queue_count == 0) {
286 		pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",
287 			 dev->id, (int)proc->lead_thread->pid);
288 		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
289 	}
290 
291 	/* Collect wave count from device if it supports */
292 	wave_cnt = 0;
293 	max_waves_per_cu = 0;
294 
295 	cu_occupancy = kcalloc(AMDGPU_MAX_QUEUES, sizeof(*cu_occupancy), GFP_KERNEL);
296 	if (!cu_occupancy)
297 		return -ENOMEM;
298 
299 	/*
300 	 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
301 	 * For AQL queues, because of cooperative dispatch we multiply the wave count
302 	 * by number of XCCs in the partition to get the total wave counts across all
303 	 * XCCs in the partition.
304 	 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
305 	 */
306 	dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
307 			&max_waves_per_cu, ffs(dev->xcc_mask) - 1);
308 
309 	for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
310 		if (cu_occupancy[i].wave_cnt != 0 &&
311 		    kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,
312 						cu_occupancy[i].doorbell_off,
313 						&queue_format)) {
314 			if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
315 				wave_cnt += cu_occupancy[i].wave_cnt;
316 			else
317 				wave_cnt += (NUM_XCC(dev->xcc_mask) *
318 						cu_occupancy[i].wave_cnt);
319 		}
320 	}
321 
322 	/* Translate wave count to number of compute units */
323 	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
324 	kfree(cu_occupancy);
325 	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
326 }
327 
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)328 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
329 			       char *buffer)
330 {
331 	if (strcmp(attr->name, "pasid") == 0)
332 		return snprintf(buffer, PAGE_SIZE, "%d\n", 0);
333 	else if (strncmp(attr->name, "vram_", 5) == 0) {
334 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
335 							      attr_vram);
336 		return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
337 	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
338 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
339 							      attr_sdma);
340 		struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
341 
342 		INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
343 				  kfd_sdma_activity_worker);
344 
345 		sdma_activity_work_handler.pdd = pdd;
346 		sdma_activity_work_handler.sdma_activity_counter = 0;
347 
348 		schedule_work(&sdma_activity_work_handler.sdma_activity_work);
349 
350 		flush_work(&sdma_activity_work_handler.sdma_activity_work);
351 		destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
352 
353 		return snprintf(buffer, PAGE_SIZE, "%llu\n",
354 				(sdma_activity_work_handler.sdma_activity_counter)/
355 				 SDMA_ACTIVITY_DIVISOR);
356 	} else {
357 		pr_err("Invalid attribute");
358 		return -EINVAL;
359 	}
360 
361 	return 0;
362 }
363 
kfd_procfs_kobj_release(struct kobject * kobj)364 static void kfd_procfs_kobj_release(struct kobject *kobj)
365 {
366 	kfree(kobj);
367 }
368 
369 static const struct sysfs_ops kfd_procfs_ops = {
370 	.show = kfd_procfs_show,
371 };
372 
373 static const struct kobj_type procfs_type = {
374 	.release = kfd_procfs_kobj_release,
375 	.sysfs_ops = &kfd_procfs_ops,
376 };
377 
kfd_procfs_init(void)378 void kfd_procfs_init(void)
379 {
380 	int ret = 0;
381 
382 	procfs.kobj = kfd_alloc_struct(procfs.kobj);
383 	if (!procfs.kobj)
384 		return;
385 
386 	ret = kobject_init_and_add(procfs.kobj, &procfs_type,
387 				   &kfd_device->kobj, "proc");
388 	if (ret) {
389 		pr_warn("Could not create procfs proc folder");
390 		/* If we fail to create the procfs, clean up */
391 		kfd_procfs_shutdown();
392 	}
393 }
394 
kfd_procfs_shutdown(void)395 void kfd_procfs_shutdown(void)
396 {
397 	if (procfs.kobj) {
398 		kobject_del(procfs.kobj);
399 		kobject_put(procfs.kobj);
400 		procfs.kobj = NULL;
401 	}
402 }
403 
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)404 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
405 				     struct attribute *attr, char *buffer)
406 {
407 	struct queue *q = container_of(kobj, struct queue, kobj);
408 
409 	if (!strcmp(attr->name, "size"))
410 		return snprintf(buffer, PAGE_SIZE, "%llu",
411 				q->properties.queue_size);
412 	else if (!strcmp(attr->name, "type"))
413 		return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
414 	else if (!strcmp(attr->name, "gpuid"))
415 		return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
416 	else
417 		pr_err("Invalid attribute");
418 
419 	return 0;
420 }
421 
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)422 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
423 				     struct attribute *attr, char *buffer)
424 {
425 	if (strcmp(attr->name, "evicted_ms") == 0) {
426 		struct kfd_process_device *pdd = container_of(attr,
427 				struct kfd_process_device,
428 				attr_evict);
429 		uint64_t evict_jiffies;
430 
431 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
432 
433 		return snprintf(buffer,
434 				PAGE_SIZE,
435 				"%llu\n",
436 				jiffies64_to_msecs(evict_jiffies));
437 
438 	/* Sysfs handle that gets CU occupancy is per device */
439 	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
440 		return kfd_get_cu_occupancy(attr, buffer);
441 	} else {
442 		pr_err("Invalid attribute");
443 	}
444 
445 	return 0;
446 }
447 
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)448 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
449 				       struct attribute *attr, char *buf)
450 {
451 	struct kfd_process_device *pdd;
452 
453 	if (!strcmp(attr->name, "faults")) {
454 		pdd = container_of(attr, struct kfd_process_device,
455 				   attr_faults);
456 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
457 	}
458 	if (!strcmp(attr->name, "page_in")) {
459 		pdd = container_of(attr, struct kfd_process_device,
460 				   attr_page_in);
461 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
462 	}
463 	if (!strcmp(attr->name, "page_out")) {
464 		pdd = container_of(attr, struct kfd_process_device,
465 				   attr_page_out);
466 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
467 	}
468 	return 0;
469 }
470 
471 static struct attribute attr_queue_size = {
472 	.name = "size",
473 	.mode = KFD_SYSFS_FILE_MODE
474 };
475 
476 static struct attribute attr_queue_type = {
477 	.name = "type",
478 	.mode = KFD_SYSFS_FILE_MODE
479 };
480 
481 static struct attribute attr_queue_gpuid = {
482 	.name = "gpuid",
483 	.mode = KFD_SYSFS_FILE_MODE
484 };
485 
486 static struct attribute *procfs_queue_attrs[] = {
487 	&attr_queue_size,
488 	&attr_queue_type,
489 	&attr_queue_gpuid,
490 	NULL
491 };
492 ATTRIBUTE_GROUPS(procfs_queue);
493 
494 static const struct sysfs_ops procfs_queue_ops = {
495 	.show = kfd_procfs_queue_show,
496 };
497 
498 static const struct kobj_type procfs_queue_type = {
499 	.sysfs_ops = &procfs_queue_ops,
500 	.default_groups = procfs_queue_groups,
501 };
502 
503 static const struct sysfs_ops procfs_stats_ops = {
504 	.show = kfd_procfs_stats_show,
505 };
506 
507 static const struct kobj_type procfs_stats_type = {
508 	.sysfs_ops = &procfs_stats_ops,
509 	.release = kfd_procfs_kobj_release,
510 };
511 
512 static const struct sysfs_ops sysfs_counters_ops = {
513 	.show = kfd_sysfs_counters_show,
514 };
515 
516 static const struct kobj_type sysfs_counters_type = {
517 	.sysfs_ops = &sysfs_counters_ops,
518 	.release = kfd_procfs_kobj_release,
519 };
520 
kfd_procfs_add_queue(struct queue * q)521 int kfd_procfs_add_queue(struct queue *q)
522 {
523 	struct kfd_process *proc;
524 	int ret;
525 
526 	if (!q || !q->process)
527 		return -EINVAL;
528 	proc = q->process;
529 
530 	/* Create proc/<pid>/queues/<queue id> folder */
531 	if (!proc->kobj_queues)
532 		return -EFAULT;
533 	ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
534 			proc->kobj_queues, "%u", q->properties.queue_id);
535 	if (ret < 0) {
536 		pr_warn("Creating proc/<pid>/queues/%u failed",
537 			q->properties.queue_id);
538 		kobject_put(&q->kobj);
539 		return ret;
540 	}
541 
542 	return 0;
543 }
544 
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)545 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
546 				 char *name)
547 {
548 	int ret;
549 
550 	if (!kobj || !attr || !name)
551 		return;
552 
553 	attr->name = name;
554 	attr->mode = KFD_SYSFS_FILE_MODE;
555 	sysfs_attr_init(attr);
556 
557 	ret = sysfs_create_file(kobj, attr);
558 	if (ret)
559 		pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
560 }
561 
kfd_procfs_add_sysfs_stats(struct kfd_process * p)562 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
563 {
564 	int ret;
565 	int i;
566 	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
567 
568 	if (!p || !p->kobj)
569 		return;
570 
571 	/*
572 	 * Create sysfs files for each GPU:
573 	 * - proc/<pid>/stats_<gpuid>/
574 	 * - proc/<pid>/stats_<gpuid>/evicted_ms
575 	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
576 	 */
577 	for (i = 0; i < p->n_pdds; i++) {
578 		struct kfd_process_device *pdd = p->pdds[i];
579 
580 		snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
581 				"stats_%u", pdd->dev->id);
582 		pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
583 		if (!pdd->kobj_stats)
584 			return;
585 
586 		ret = kobject_init_and_add(pdd->kobj_stats,
587 					   &procfs_stats_type,
588 					   p->kobj,
589 					   stats_dir_filename);
590 
591 		if (ret) {
592 			pr_warn("Creating KFD proc/stats_%s folder failed",
593 				stats_dir_filename);
594 			kobject_put(pdd->kobj_stats);
595 			pdd->kobj_stats = NULL;
596 			return;
597 		}
598 
599 		kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
600 				      "evicted_ms");
601 		/* Add sysfs file to report compute unit occupancy */
602 		if (pdd->dev->kfd2kgd->get_cu_occupancy)
603 			kfd_sysfs_create_file(pdd->kobj_stats,
604 					      &pdd->attr_cu_occupancy,
605 					      "cu_occupancy");
606 	}
607 }
608 
kfd_procfs_add_sysfs_counters(struct kfd_process * p)609 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
610 {
611 	int ret = 0;
612 	int i;
613 	char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
614 
615 	if (!p || !p->kobj)
616 		return;
617 
618 	/*
619 	 * Create sysfs files for each GPU which supports SVM
620 	 * - proc/<pid>/counters_<gpuid>/
621 	 * - proc/<pid>/counters_<gpuid>/faults
622 	 * - proc/<pid>/counters_<gpuid>/page_in
623 	 * - proc/<pid>/counters_<gpuid>/page_out
624 	 */
625 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
626 		struct kfd_process_device *pdd = p->pdds[i];
627 		struct kobject *kobj_counters;
628 
629 		snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
630 			"counters_%u", pdd->dev->id);
631 		kobj_counters = kfd_alloc_struct(kobj_counters);
632 		if (!kobj_counters)
633 			return;
634 
635 		ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
636 					   p->kobj, counters_dir_filename);
637 		if (ret) {
638 			pr_warn("Creating KFD proc/%s folder failed",
639 				counters_dir_filename);
640 			kobject_put(kobj_counters);
641 			return;
642 		}
643 
644 		pdd->kobj_counters = kobj_counters;
645 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
646 				      "faults");
647 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
648 				      "page_in");
649 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
650 				      "page_out");
651 	}
652 }
653 
kfd_procfs_add_sysfs_files(struct kfd_process * p)654 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
655 {
656 	int i;
657 
658 	if (!p || !p->kobj)
659 		return;
660 
661 	/*
662 	 * Create sysfs files for each GPU:
663 	 * - proc/<pid>/vram_<gpuid>
664 	 * - proc/<pid>/sdma_<gpuid>
665 	 */
666 	for (i = 0; i < p->n_pdds; i++) {
667 		struct kfd_process_device *pdd = p->pdds[i];
668 
669 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
670 			 pdd->dev->id);
671 		kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
672 				      pdd->vram_filename);
673 
674 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
675 			 pdd->dev->id);
676 		kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
677 					    pdd->sdma_filename);
678 	}
679 }
680 
kfd_procfs_del_queue(struct queue * q)681 void kfd_procfs_del_queue(struct queue *q)
682 {
683 	if (!q)
684 		return;
685 
686 	kobject_del(&q->kobj);
687 	kobject_put(&q->kobj);
688 }
689 
kfd_process_create_wq(void)690 int kfd_process_create_wq(void)
691 {
692 	if (!kfd_process_wq)
693 		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
694 	if (!kfd_restore_wq)
695 		kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
696 							 WQ_FREEZABLE);
697 
698 	if (!kfd_process_wq || !kfd_restore_wq) {
699 		kfd_process_destroy_wq();
700 		return -ENOMEM;
701 	}
702 
703 	return 0;
704 }
705 
kfd_process_destroy_wq(void)706 void kfd_process_destroy_wq(void)
707 {
708 	if (kfd_process_wq) {
709 		destroy_workqueue(kfd_process_wq);
710 		kfd_process_wq = NULL;
711 	}
712 	if (kfd_restore_wq) {
713 		destroy_workqueue(kfd_restore_wq);
714 		kfd_restore_wq = NULL;
715 	}
716 }
717 
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd,void ** kptr)718 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
719 			struct kfd_process_device *pdd, void **kptr)
720 {
721 	struct kfd_node *dev = pdd->dev;
722 
723 	if (kptr && *kptr) {
724 		amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
725 		*kptr = NULL;
726 	}
727 
728 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
729 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
730 					       NULL);
731 }
732 
733 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
734  *	This function should be only called right after the process
735  *	is created and when kfd_processes_mutex is still being held
736  *	to avoid concurrency. Because of that exclusiveness, we do
737  *	not need to take p->mutex.
738  */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,struct kgd_mem ** mem,void ** kptr)739 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
740 				   uint64_t gpu_va, uint32_t size,
741 				   uint32_t flags, struct kgd_mem **mem, void **kptr)
742 {
743 	struct kfd_node *kdev = pdd->dev;
744 	int err;
745 
746 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
747 						 pdd->drm_priv, mem, NULL,
748 						 flags, false);
749 	if (err)
750 		goto err_alloc_mem;
751 
752 	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
753 			pdd->drm_priv);
754 	if (err)
755 		goto err_map_mem;
756 
757 	err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
758 	if (err) {
759 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
760 		goto sync_memory_failed;
761 	}
762 
763 	if (kptr) {
764 		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
765 				(struct kgd_mem *)*mem, kptr, NULL);
766 		if (err) {
767 			pr_debug("Map GTT BO to kernel failed\n");
768 			goto sync_memory_failed;
769 		}
770 	}
771 
772 	return err;
773 
774 sync_memory_failed:
775 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
776 
777 err_map_mem:
778 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
779 					       NULL);
780 err_alloc_mem:
781 	*mem = NULL;
782 	*kptr = NULL;
783 	return err;
784 }
785 
786 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
787  *	process for IB usage The memory reserved is for KFD to submit
788  *	IB to AMDGPU from kernel.  If the memory is reserved
789  *	successfully, ib_kaddr will have the CPU/kernel
790  *	address. Check ib_kaddr before accessing the memory.
791  */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)792 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
793 {
794 	struct qcm_process_device *qpd = &pdd->qpd;
795 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
796 			KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
797 			KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
798 			KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
799 	struct kgd_mem *mem;
800 	void *kaddr;
801 	int ret;
802 
803 	if (qpd->ib_kaddr || !qpd->ib_base)
804 		return 0;
805 
806 	/* ib_base is only set for dGPU */
807 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
808 				      &mem, &kaddr);
809 	if (ret)
810 		return ret;
811 
812 	qpd->ib_mem = mem;
813 	qpd->ib_kaddr = kaddr;
814 
815 	return 0;
816 }
817 
kfd_process_device_destroy_ib_mem(struct kfd_process_device * pdd)818 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
819 {
820 	struct qcm_process_device *qpd = &pdd->qpd;
821 
822 	if (!qpd->ib_kaddr || !qpd->ib_base)
823 		return;
824 
825 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
826 }
827 
kfd_create_process(struct task_struct * thread)828 struct kfd_process *kfd_create_process(struct task_struct *thread)
829 {
830 	struct kfd_process *process;
831 	int ret;
832 
833 	if (!(thread->mm && mmget_not_zero(thread->mm)))
834 		return ERR_PTR(-EINVAL);
835 
836 	/* Only the pthreads threading model is supported. */
837 	if (thread->group_leader->mm != thread->mm) {
838 		mmput(thread->mm);
839 		return ERR_PTR(-EINVAL);
840 	}
841 
842 	/* If the process just called exec(3), it is possible that the
843 	 * cleanup of the kfd_process (following the release of the mm
844 	 * of the old process image) is still in the cleanup work queue.
845 	 * Make sure to drain any job before trying to recreate any
846 	 * resource for this process.
847 	 */
848 	flush_workqueue(kfd_process_wq);
849 
850 	/*
851 	 * take kfd processes mutex before starting of process creation
852 	 * so there won't be a case where two threads of the same process
853 	 * create two kfd_process structures
854 	 */
855 	mutex_lock(&kfd_processes_mutex);
856 
857 	if (kfd_is_locked()) {
858 		pr_debug("KFD is locked! Cannot create process");
859 		process = ERR_PTR(-EINVAL);
860 		goto out;
861 	}
862 
863 	/* A prior open of /dev/kfd could have already created the process.
864 	 * find_process will increase process kref in this case
865 	 */
866 	process = find_process(thread, true);
867 	if (process) {
868 		pr_debug("Process already found\n");
869 	} else {
870 		process = create_process(thread);
871 		if (IS_ERR(process))
872 			goto out;
873 
874 		if (!procfs.kobj)
875 			goto out;
876 
877 		process->kobj = kfd_alloc_struct(process->kobj);
878 		if (!process->kobj) {
879 			pr_warn("Creating procfs kobject failed");
880 			goto out;
881 		}
882 		ret = kobject_init_and_add(process->kobj, &procfs_type,
883 					   procfs.kobj, "%d",
884 					   (int)process->lead_thread->pid);
885 		if (ret) {
886 			pr_warn("Creating procfs pid directory failed");
887 			kobject_put(process->kobj);
888 			goto out;
889 		}
890 
891 		kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
892 				      "pasid");
893 
894 		process->kobj_queues = kobject_create_and_add("queues",
895 							process->kobj);
896 		if (!process->kobj_queues)
897 			pr_warn("Creating KFD proc/queues folder failed");
898 
899 		kfd_procfs_add_sysfs_stats(process);
900 		kfd_procfs_add_sysfs_files(process);
901 		kfd_procfs_add_sysfs_counters(process);
902 
903 		init_waitqueue_head(&process->wait_irq_drain);
904 	}
905 out:
906 	mutex_unlock(&kfd_processes_mutex);
907 	mmput(thread->mm);
908 
909 	return process;
910 }
911 
kfd_get_process(const struct task_struct * thread)912 struct kfd_process *kfd_get_process(const struct task_struct *thread)
913 {
914 	struct kfd_process *process;
915 
916 	if (!thread->mm)
917 		return ERR_PTR(-EINVAL);
918 
919 	/* Only the pthreads threading model is supported. */
920 	if (thread->group_leader->mm != thread->mm)
921 		return ERR_PTR(-EINVAL);
922 
923 	process = find_process(thread, false);
924 	if (!process)
925 		return ERR_PTR(-EINVAL);
926 
927 	return process;
928 }
929 
find_process_by_mm(const struct mm_struct * mm)930 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
931 {
932 	struct kfd_process *process;
933 
934 	hash_for_each_possible_rcu(kfd_processes_table, process,
935 					kfd_processes, (uintptr_t)mm)
936 		if (process->mm == mm)
937 			return process;
938 
939 	return NULL;
940 }
941 
find_process(const struct task_struct * thread,bool ref)942 static struct kfd_process *find_process(const struct task_struct *thread,
943 					bool ref)
944 {
945 	struct kfd_process *p;
946 	int idx;
947 
948 	idx = srcu_read_lock(&kfd_processes_srcu);
949 	p = find_process_by_mm(thread->mm);
950 	if (p && ref)
951 		kref_get(&p->ref);
952 	srcu_read_unlock(&kfd_processes_srcu, idx);
953 
954 	return p;
955 }
956 
kfd_unref_process(struct kfd_process * p)957 void kfd_unref_process(struct kfd_process *p)
958 {
959 	kref_put(&p->ref, kfd_process_ref_release);
960 }
961 
962 /* This increments the process->ref counter. */
kfd_lookup_process_by_pid(struct pid * pid)963 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
964 {
965 	struct task_struct *task = NULL;
966 	struct kfd_process *p    = NULL;
967 
968 	if (!pid) {
969 		task = current;
970 		get_task_struct(task);
971 	} else {
972 		task = get_pid_task(pid, PIDTYPE_PID);
973 	}
974 
975 	if (task) {
976 		p = find_process(task, true);
977 		put_task_struct(task);
978 	}
979 
980 	return p;
981 }
982 
kfd_process_device_free_bos(struct kfd_process_device * pdd)983 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
984 {
985 	struct kfd_process *p = pdd->process;
986 	void *mem;
987 	int id;
988 	int i;
989 
990 	/*
991 	 * Remove all handles from idr and release appropriate
992 	 * local memory object
993 	 */
994 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
995 
996 		for (i = 0; i < p->n_pdds; i++) {
997 			struct kfd_process_device *peer_pdd = p->pdds[i];
998 
999 			if (!peer_pdd->drm_priv)
1000 				continue;
1001 			amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1002 				peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
1003 		}
1004 
1005 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
1006 						       pdd->drm_priv, NULL);
1007 		kfd_process_device_remove_obj_handle(pdd, id);
1008 	}
1009 }
1010 
1011 /*
1012  * Just kunmap and unpin signal BO here. It will be freed in
1013  * kfd_process_free_outstanding_kfd_bos()
1014  */
kfd_process_kunmap_signal_bo(struct kfd_process * p)1015 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1016 {
1017 	struct kfd_process_device *pdd;
1018 	struct kfd_node *kdev;
1019 	void *mem;
1020 
1021 	kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1022 	if (!kdev)
1023 		return;
1024 
1025 	mutex_lock(&p->mutex);
1026 
1027 	pdd = kfd_get_process_device_data(kdev, p);
1028 	if (!pdd)
1029 		goto out;
1030 
1031 	mem = kfd_process_device_translate_handle(
1032 		pdd, GET_IDR_HANDLE(p->signal_handle));
1033 	if (!mem)
1034 		goto out;
1035 
1036 	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1037 
1038 out:
1039 	mutex_unlock(&p->mutex);
1040 }
1041 
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)1042 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1043 {
1044 	int i;
1045 
1046 	for (i = 0; i < p->n_pdds; i++)
1047 		kfd_process_device_free_bos(p->pdds[i]);
1048 }
1049 
kfd_process_destroy_pdds(struct kfd_process * p)1050 static void kfd_process_destroy_pdds(struct kfd_process *p)
1051 {
1052 	int i;
1053 
1054 	for (i = 0; i < p->n_pdds; i++) {
1055 		struct kfd_process_device *pdd = p->pdds[i];
1056 
1057 		pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
1058 			pdd->dev->id, p->lead_thread->pid);
1059 		kfd_process_device_destroy_cwsr_dgpu(pdd);
1060 		kfd_process_device_destroy_ib_mem(pdd);
1061 
1062 		if (pdd->drm_file)
1063 			fput(pdd->drm_file);
1064 
1065 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1066 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1067 				get_order(KFD_CWSR_TBA_TMA_SIZE));
1068 
1069 		idr_destroy(&pdd->alloc_idr);
1070 
1071 		kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1072 
1073 		if (pdd->dev->kfd->shared_resources.enable_mes &&
1074 			pdd->proc_ctx_cpu_ptr)
1075 			amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
1076 						   &pdd->proc_ctx_bo);
1077 		/*
1078 		 * before destroying pdd, make sure to report availability
1079 		 * for auto suspend
1080 		 */
1081 		if (pdd->runtime_inuse) {
1082 			pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);
1083 			pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1084 			pdd->runtime_inuse = false;
1085 		}
1086 
1087 		kfree(pdd);
1088 		p->pdds[i] = NULL;
1089 	}
1090 	p->n_pdds = 0;
1091 }
1092 
kfd_process_remove_sysfs(struct kfd_process * p)1093 static void kfd_process_remove_sysfs(struct kfd_process *p)
1094 {
1095 	struct kfd_process_device *pdd;
1096 	int i;
1097 
1098 	if (!p->kobj)
1099 		return;
1100 
1101 	sysfs_remove_file(p->kobj, &p->attr_pasid);
1102 	kobject_del(p->kobj_queues);
1103 	kobject_put(p->kobj_queues);
1104 	p->kobj_queues = NULL;
1105 
1106 	for (i = 0; i < p->n_pdds; i++) {
1107 		pdd = p->pdds[i];
1108 
1109 		sysfs_remove_file(p->kobj, &pdd->attr_vram);
1110 		sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1111 
1112 		sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1113 		if (pdd->dev->kfd2kgd->get_cu_occupancy)
1114 			sysfs_remove_file(pdd->kobj_stats,
1115 					  &pdd->attr_cu_occupancy);
1116 		kobject_del(pdd->kobj_stats);
1117 		kobject_put(pdd->kobj_stats);
1118 		pdd->kobj_stats = NULL;
1119 	}
1120 
1121 	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1122 		pdd = p->pdds[i];
1123 
1124 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1125 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1126 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1127 		kobject_del(pdd->kobj_counters);
1128 		kobject_put(pdd->kobj_counters);
1129 		pdd->kobj_counters = NULL;
1130 	}
1131 
1132 	kobject_del(p->kobj);
1133 	kobject_put(p->kobj);
1134 	p->kobj = NULL;
1135 }
1136 
1137 /*
1138  * If any GPU is ongoing reset, wait for reset complete.
1139  */
kfd_process_wait_gpu_reset_complete(struct kfd_process * p)1140 static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1141 {
1142 	int i;
1143 
1144 	for (i = 0; i < p->n_pdds; i++)
1145 		flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1146 }
1147 
1148 /* No process locking is needed in this function, because the process
1149  * is not findable any more. We must assume that no other thread is
1150  * using it any more, otherwise we couldn't safely free the process
1151  * structure in the end.
1152  */
kfd_process_wq_release(struct work_struct * work)1153 static void kfd_process_wq_release(struct work_struct *work)
1154 {
1155 	struct kfd_process *p = container_of(work, struct kfd_process,
1156 					     release_work);
1157 	struct dma_fence *ef;
1158 
1159 	kfd_process_dequeue_from_all_devices(p);
1160 	pqm_uninit(&p->pqm);
1161 
1162 	/*
1163 	 * If GPU in reset, user queues may still running, wait for reset complete.
1164 	 */
1165 	kfd_process_wait_gpu_reset_complete(p);
1166 
1167 	/* Signal the eviction fence after user mode queues are
1168 	 * destroyed. This allows any BOs to be freed without
1169 	 * triggering pointless evictions or waiting for fences.
1170 	 */
1171 	synchronize_rcu();
1172 	ef = rcu_access_pointer(p->ef);
1173 	if (ef)
1174 		dma_fence_signal(ef);
1175 
1176 	kfd_process_remove_sysfs(p);
1177 
1178 	kfd_process_kunmap_signal_bo(p);
1179 	kfd_process_free_outstanding_kfd_bos(p);
1180 	svm_range_list_fini(p);
1181 
1182 	kfd_process_destroy_pdds(p);
1183 	dma_fence_put(ef);
1184 
1185 	kfd_event_free_process(p);
1186 
1187 	mutex_destroy(&p->mutex);
1188 
1189 	put_task_struct(p->lead_thread);
1190 
1191 	kfree(p);
1192 }
1193 
kfd_process_ref_release(struct kref * ref)1194 static void kfd_process_ref_release(struct kref *ref)
1195 {
1196 	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1197 
1198 	INIT_WORK(&p->release_work, kfd_process_wq_release);
1199 	queue_work(kfd_process_wq, &p->release_work);
1200 }
1201 
kfd_process_alloc_notifier(struct mm_struct * mm)1202 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1203 {
1204 	/* This increments p->ref counter if kfd process p exists */
1205 	struct kfd_process *p = kfd_lookup_process_by_mm(mm);
1206 
1207 	return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1208 }
1209 
kfd_process_free_notifier(struct mmu_notifier * mn)1210 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1211 {
1212 	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1213 }
1214 
kfd_process_notifier_release_internal(struct kfd_process * p)1215 static void kfd_process_notifier_release_internal(struct kfd_process *p)
1216 {
1217 	int i;
1218 
1219 	cancel_delayed_work_sync(&p->eviction_work);
1220 	cancel_delayed_work_sync(&p->restore_work);
1221 
1222 	for (i = 0; i < p->n_pdds; i++) {
1223 		struct kfd_process_device *pdd = p->pdds[i];
1224 
1225 		/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1226 		if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1227 			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1228 	}
1229 
1230 	/* Indicate to other users that MM is no longer valid */
1231 	p->mm = NULL;
1232 	kfd_dbg_trap_disable(p);
1233 
1234 	if (atomic_read(&p->debugged_process_count) > 0) {
1235 		struct kfd_process *target;
1236 		unsigned int temp;
1237 		int idx = srcu_read_lock(&kfd_processes_srcu);
1238 
1239 		hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1240 			if (target->debugger_process && target->debugger_process == p) {
1241 				mutex_lock_nested(&target->mutex, 1);
1242 				kfd_dbg_trap_disable(target);
1243 				mutex_unlock(&target->mutex);
1244 				if (atomic_read(&p->debugged_process_count) == 0)
1245 					break;
1246 			}
1247 		}
1248 
1249 		srcu_read_unlock(&kfd_processes_srcu, idx);
1250 	}
1251 
1252 	mmu_notifier_put(&p->mmu_notifier);
1253 }
1254 
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1255 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1256 					struct mm_struct *mm)
1257 {
1258 	struct kfd_process *p;
1259 
1260 	/*
1261 	 * The kfd_process structure can not be free because the
1262 	 * mmu_notifier srcu is read locked
1263 	 */
1264 	p = container_of(mn, struct kfd_process, mmu_notifier);
1265 	if (WARN_ON(p->mm != mm))
1266 		return;
1267 
1268 	mutex_lock(&kfd_processes_mutex);
1269 	/*
1270 	 * Do early return if table is empty.
1271 	 *
1272 	 * This could potentially happen if this function is called concurrently
1273 	 * by mmu_notifier and by kfd_cleanup_pocesses.
1274 	 *
1275 	 */
1276 	if (hash_empty(kfd_processes_table)) {
1277 		mutex_unlock(&kfd_processes_mutex);
1278 		return;
1279 	}
1280 	hash_del_rcu(&p->kfd_processes);
1281 	mutex_unlock(&kfd_processes_mutex);
1282 	synchronize_srcu(&kfd_processes_srcu);
1283 
1284 	kfd_process_notifier_release_internal(p);
1285 }
1286 
1287 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1288 	.release = kfd_process_notifier_release,
1289 	.alloc_notifier = kfd_process_alloc_notifier,
1290 	.free_notifier = kfd_process_free_notifier,
1291 };
1292 
1293 /*
1294  * This code handles the case when driver is being unloaded before all
1295  * mm_struct are released.  We need to safely free the kfd_process and
1296  * avoid race conditions with mmu_notifier that might try to free them.
1297  *
1298  */
kfd_cleanup_processes(void)1299 void kfd_cleanup_processes(void)
1300 {
1301 	struct kfd_process *p;
1302 	struct hlist_node *p_temp;
1303 	unsigned int temp;
1304 	HLIST_HEAD(cleanup_list);
1305 
1306 	/*
1307 	 * Move all remaining kfd_process from the process table to a
1308 	 * temp list for processing.   Once done, callback from mmu_notifier
1309 	 * release will not see the kfd_process in the table and do early return,
1310 	 * avoiding double free issues.
1311 	 */
1312 	mutex_lock(&kfd_processes_mutex);
1313 	hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1314 		hash_del_rcu(&p->kfd_processes);
1315 		synchronize_srcu(&kfd_processes_srcu);
1316 		hlist_add_head(&p->kfd_processes, &cleanup_list);
1317 	}
1318 	mutex_unlock(&kfd_processes_mutex);
1319 
1320 	hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1321 		kfd_process_notifier_release_internal(p);
1322 
1323 	/*
1324 	 * Ensures that all outstanding free_notifier get called, triggering
1325 	 * the release of the kfd_process struct.
1326 	 */
1327 	mmu_notifier_synchronize();
1328 }
1329 
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1330 int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1331 {
1332 	unsigned long  offset;
1333 	int i;
1334 
1335 	if (p->has_cwsr)
1336 		return 0;
1337 
1338 	for (i = 0; i < p->n_pdds; i++) {
1339 		struct kfd_node *dev = p->pdds[i]->dev;
1340 		struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1341 
1342 		if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1343 			continue;
1344 
1345 		offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1346 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1347 			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1348 			MAP_SHARED, offset);
1349 
1350 		if (IS_ERR_VALUE(qpd->tba_addr)) {
1351 			int err = qpd->tba_addr;
1352 
1353 			dev_err(dev->adev->dev,
1354 				"Failure to set tba address. error %d.\n", err);
1355 			qpd->tba_addr = 0;
1356 			qpd->cwsr_kaddr = NULL;
1357 			return err;
1358 		}
1359 
1360 		memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1361 
1362 		kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
1363 
1364 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1365 		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1366 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1367 	}
1368 
1369 	p->has_cwsr = true;
1370 
1371 	return 0;
1372 }
1373 
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1374 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1375 {
1376 	struct kfd_node *dev = pdd->dev;
1377 	struct qcm_process_device *qpd = &pdd->qpd;
1378 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1379 			| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1380 			| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1381 	struct kgd_mem *mem;
1382 	void *kaddr;
1383 	int ret;
1384 
1385 	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1386 		return 0;
1387 
1388 	/* cwsr_base is only set for dGPU */
1389 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1390 				      KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1391 	if (ret)
1392 		return ret;
1393 
1394 	qpd->cwsr_mem = mem;
1395 	qpd->cwsr_kaddr = kaddr;
1396 	qpd->tba_addr = qpd->cwsr_base;
1397 
1398 	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1399 
1400 	kfd_process_set_trap_debug_flag(&pdd->qpd,
1401 					pdd->process->debug_trap_enabled);
1402 
1403 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1404 	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1405 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1406 
1407 	return 0;
1408 }
1409 
kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device * pdd)1410 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1411 {
1412 	struct kfd_node *dev = pdd->dev;
1413 	struct qcm_process_device *qpd = &pdd->qpd;
1414 
1415 	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1416 		return;
1417 
1418 	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
1419 }
1420 
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1421 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1422 				  uint64_t tba_addr,
1423 				  uint64_t tma_addr)
1424 {
1425 	if (qpd->cwsr_kaddr) {
1426 		/* KFD trap handler is bound, record as second-level TBA/TMA
1427 		 * in first-level TMA. First-level trap will jump to second.
1428 		 */
1429 		uint64_t *tma =
1430 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1431 		tma[0] = tba_addr;
1432 		tma[1] = tma_addr;
1433 	} else {
1434 		/* No trap handler bound, bind as first-level TBA/TMA. */
1435 		qpd->tba_addr = tba_addr;
1436 		qpd->tma_addr = tma_addr;
1437 	}
1438 }
1439 
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1440 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1441 {
1442 	int i;
1443 
1444 	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
1445 	 * boot time retry setting. Mixing processes with different
1446 	 * XNACK/retry settings can hang the GPU.
1447 	 *
1448 	 * Different GPUs can have different noretry settings depending
1449 	 * on HW bugs or limitations. We need to find at least one
1450 	 * XNACK mode for this process that's compatible with all GPUs.
1451 	 * Fortunately GPUs with retry enabled (noretry=0) can run code
1452 	 * built for XNACK-off. On GFXv9 it may perform slower.
1453 	 *
1454 	 * Therefore applications built for XNACK-off can always be
1455 	 * supported and will be our fallback if any GPU does not
1456 	 * support retry.
1457 	 */
1458 	for (i = 0; i < p->n_pdds; i++) {
1459 		struct kfd_node *dev = p->pdds[i]->dev;
1460 
1461 		/* Only consider GFXv9 and higher GPUs. Older GPUs don't
1462 		 * support the SVM APIs and don't need to be considered
1463 		 * for the XNACK mode selection.
1464 		 */
1465 		if (!KFD_IS_SOC15(dev))
1466 			continue;
1467 		/* Aldebaran can always support XNACK because it can support
1468 		 * per-process XNACK mode selection. But let the dev->noretry
1469 		 * setting still influence the default XNACK mode.
1470 		 */
1471 		if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1472 			if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1473 				pr_debug("SRIOV platform xnack not supported\n");
1474 				return false;
1475 			}
1476 			continue;
1477 		}
1478 
1479 		/* GFXv10 and later GPUs do not support shader preemption
1480 		 * during page faults. This can lead to poor QoS for queue
1481 		 * management and memory-manager-related preemptions or
1482 		 * even deadlocks.
1483 		 */
1484 		if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1485 			return false;
1486 
1487 		if (dev->kfd->noretry)
1488 			return false;
1489 	}
1490 
1491 	return true;
1492 }
1493 
kfd_process_set_trap_debug_flag(struct qcm_process_device * qpd,bool enabled)1494 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1495 				     bool enabled)
1496 {
1497 	if (qpd->cwsr_kaddr) {
1498 		uint64_t *tma =
1499 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1500 		tma[2] = enabled;
1501 	}
1502 }
1503 
1504 /*
1505  * On return the kfd_process is fully operational and will be freed when the
1506  * mm is released
1507  */
create_process(const struct task_struct * thread)1508 static struct kfd_process *create_process(const struct task_struct *thread)
1509 {
1510 	struct kfd_process *process;
1511 	struct mmu_notifier *mn;
1512 	int err = -ENOMEM;
1513 
1514 	process = kzalloc(sizeof(*process), GFP_KERNEL);
1515 	if (!process)
1516 		goto err_alloc_process;
1517 
1518 	kref_init(&process->ref);
1519 	mutex_init(&process->mutex);
1520 	process->mm = thread->mm;
1521 	process->lead_thread = thread->group_leader;
1522 	process->n_pdds = 0;
1523 	process->queues_paused = false;
1524 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1525 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1526 	process->last_restore_timestamp = get_jiffies_64();
1527 	err = kfd_event_init_process(process);
1528 	if (err)
1529 		goto err_event_init;
1530 	process->is_32bit_user_mode = in_compat_syscall();
1531 	process->debug_trap_enabled = false;
1532 	process->debugger_process = NULL;
1533 	process->exception_enable_mask = 0;
1534 	atomic_set(&process->debugged_process_count, 0);
1535 	sema_init(&process->runtime_enable_sema, 0);
1536 
1537 	err = pqm_init(&process->pqm, process);
1538 	if (err != 0)
1539 		goto err_process_pqm_init;
1540 
1541 	/* init process apertures*/
1542 	err = kfd_init_apertures(process);
1543 	if (err != 0)
1544 		goto err_init_apertures;
1545 
1546 	/* Check XNACK support after PDDs are created in kfd_init_apertures */
1547 	process->xnack_enabled = kfd_process_xnack_mode(process, false);
1548 
1549 	err = svm_range_list_init(process);
1550 	if (err)
1551 		goto err_init_svm_range_list;
1552 
1553 	/* alloc_notifier needs to find the process in the hash table */
1554 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1555 			(uintptr_t)process->mm);
1556 
1557 	/* Avoid free_notifier to start kfd_process_wq_release if
1558 	 * mmu_notifier_get failed because of pending signal.
1559 	 */
1560 	kref_get(&process->ref);
1561 
1562 	/* MMU notifier registration must be the last call that can fail
1563 	 * because after this point we cannot unwind the process creation.
1564 	 * After this point, mmu_notifier_put will trigger the cleanup by
1565 	 * dropping the last process reference in the free_notifier.
1566 	 */
1567 	mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1568 	if (IS_ERR(mn)) {
1569 		err = PTR_ERR(mn);
1570 		goto err_register_notifier;
1571 	}
1572 	BUG_ON(mn != &process->mmu_notifier);
1573 
1574 	kfd_unref_process(process);
1575 	get_task_struct(process->lead_thread);
1576 
1577 	INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1578 
1579 	return process;
1580 
1581 err_register_notifier:
1582 	hash_del_rcu(&process->kfd_processes);
1583 	svm_range_list_fini(process);
1584 err_init_svm_range_list:
1585 	kfd_process_free_outstanding_kfd_bos(process);
1586 	kfd_process_destroy_pdds(process);
1587 err_init_apertures:
1588 	pqm_uninit(&process->pqm);
1589 err_process_pqm_init:
1590 	kfd_event_free_process(process);
1591 err_event_init:
1592 	mutex_destroy(&process->mutex);
1593 	kfree(process);
1594 err_alloc_process:
1595 	return ERR_PTR(err);
1596 }
1597 
kfd_get_process_device_data(struct kfd_node * dev,struct kfd_process * p)1598 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1599 							struct kfd_process *p)
1600 {
1601 	int i;
1602 
1603 	for (i = 0; i < p->n_pdds; i++)
1604 		if (p->pdds[i]->dev == dev)
1605 			return p->pdds[i];
1606 
1607 	return NULL;
1608 }
1609 
kfd_create_process_device_data(struct kfd_node * dev,struct kfd_process * p)1610 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1611 							struct kfd_process *p)
1612 {
1613 	struct kfd_process_device *pdd = NULL;
1614 
1615 	if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1616 		return NULL;
1617 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1618 	if (!pdd)
1619 		return NULL;
1620 
1621 	pdd->dev = dev;
1622 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
1623 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1624 	pdd->qpd.dqm = dev->dqm;
1625 	pdd->qpd.pqm = &p->pqm;
1626 	pdd->qpd.evicted = 0;
1627 	pdd->qpd.mapped_gws_queue = false;
1628 	pdd->process = p;
1629 	pdd->bound = PDD_UNBOUND;
1630 	pdd->already_dequeued = false;
1631 	pdd->runtime_inuse = false;
1632 	atomic64_set(&pdd->vram_usage, 0);
1633 	pdd->sdma_past_activity_counter = 0;
1634 	pdd->user_gpu_id = dev->id;
1635 	atomic64_set(&pdd->evict_duration_counter, 0);
1636 
1637 	p->pdds[p->n_pdds++] = pdd;
1638 	if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1639 		pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1640 							pdd->dev->adev,
1641 							false,
1642 							0);
1643 
1644 	/* Init idr used for memory handle translation */
1645 	idr_init(&pdd->alloc_idr);
1646 
1647 	return pdd;
1648 }
1649 
1650 /**
1651  * kfd_process_device_init_vm - Initialize a VM for a process-device
1652  *
1653  * @pdd: The process-device
1654  * @drm_file: Optional pointer to a DRM file descriptor
1655  *
1656  * If @drm_file is specified, it will be used to acquire the VM from
1657  * that file descriptor. If successful, the @pdd takes ownership of
1658  * the file descriptor.
1659  *
1660  * If @drm_file is NULL, a new VM is created.
1661  *
1662  * Returns 0 on success, -errno on failure.
1663  */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1664 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1665 			       struct file *drm_file)
1666 {
1667 	struct amdgpu_fpriv *drv_priv;
1668 	struct amdgpu_vm *avm;
1669 	struct kfd_process *p;
1670 	struct dma_fence *ef;
1671 	struct kfd_node *dev;
1672 	int ret;
1673 
1674 	if (!drm_file)
1675 		return -EINVAL;
1676 
1677 	if (pdd->drm_priv)
1678 		return -EBUSY;
1679 
1680 	ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1681 	if (ret)
1682 		return ret;
1683 	avm = &drv_priv->vm;
1684 
1685 	p = pdd->process;
1686 	dev = pdd->dev;
1687 
1688 	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1689 						     &p->kgd_process_info,
1690 						     p->ef ? NULL : &ef);
1691 	if (ret) {
1692 		dev_err(dev->adev->dev, "Failed to create process VM object\n");
1693 		return ret;
1694 	}
1695 
1696 	if (!p->ef)
1697 		RCU_INIT_POINTER(p->ef, ef);
1698 
1699 	pdd->drm_priv = drm_file->private_data;
1700 
1701 	ret = kfd_process_device_reserve_ib_mem(pdd);
1702 	if (ret)
1703 		goto err_reserve_ib_mem;
1704 	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1705 	if (ret)
1706 		goto err_init_cwsr;
1707 
1708 	if (unlikely(!avm->pasid)) {
1709 		dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",
1710 				 avm);
1711 		ret = -EINVAL;
1712 		goto err_get_pasid;
1713 	}
1714 
1715 	pdd->pasid = avm->pasid;
1716 	pdd->drm_file = drm_file;
1717 
1718 	return 0;
1719 
1720 err_get_pasid:
1721 	kfd_process_device_destroy_cwsr_dgpu(pdd);
1722 err_init_cwsr:
1723 	kfd_process_device_destroy_ib_mem(pdd);
1724 err_reserve_ib_mem:
1725 	pdd->drm_priv = NULL;
1726 	amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1727 
1728 	return ret;
1729 }
1730 
1731 /*
1732  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1733  * to the device.
1734  * Unbinding occurs when the process dies or the device is removed.
1735  *
1736  * Assumes that the process lock is held.
1737  */
kfd_bind_process_to_device(struct kfd_node * dev,struct kfd_process * p)1738 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1739 							struct kfd_process *p)
1740 {
1741 	struct kfd_process_device *pdd;
1742 	int err;
1743 
1744 	pdd = kfd_get_process_device_data(dev, p);
1745 	if (!pdd) {
1746 		dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1747 		return ERR_PTR(-ENOMEM);
1748 	}
1749 
1750 	if (!pdd->drm_priv)
1751 		return ERR_PTR(-ENODEV);
1752 
1753 	/*
1754 	 * signal runtime-pm system to auto resume and prevent
1755 	 * further runtime suspend once device pdd is created until
1756 	 * pdd is destroyed.
1757 	 */
1758 	if (!pdd->runtime_inuse) {
1759 		err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1760 		if (err < 0) {
1761 			pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1762 			return ERR_PTR(err);
1763 		}
1764 	}
1765 
1766 	/*
1767 	 * make sure that runtime_usage counter is incremented just once
1768 	 * per pdd
1769 	 */
1770 	pdd->runtime_inuse = true;
1771 
1772 	return pdd;
1773 }
1774 
1775 /* Create specific handle mapped to mem from process local memory idr
1776  * Assumes that the process lock is held.
1777  */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1778 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1779 					void *mem)
1780 {
1781 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1782 }
1783 
1784 /* Translate specific handle from process local memory idr
1785  * Assumes that the process lock is held.
1786  */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1787 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1788 					int handle)
1789 {
1790 	if (handle < 0)
1791 		return NULL;
1792 
1793 	return idr_find(&pdd->alloc_idr, handle);
1794 }
1795 
1796 /* Remove specific handle from process local memory idr
1797  * Assumes that the process lock is held.
1798  */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1799 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1800 					int handle)
1801 {
1802 	if (handle >= 0)
1803 		idr_remove(&pdd->alloc_idr, handle);
1804 }
1805 
kfd_lookup_process_device_by_pasid(u32 pasid)1806 static struct kfd_process_device *kfd_lookup_process_device_by_pasid(u32 pasid)
1807 {
1808 	struct kfd_process_device *ret_p = NULL;
1809 	struct kfd_process *p;
1810 	unsigned int temp;
1811 	int i;
1812 
1813 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1814 		for (i = 0; i < p->n_pdds; i++) {
1815 			if (p->pdds[i]->pasid == pasid) {
1816 				ret_p = p->pdds[i];
1817 				break;
1818 			}
1819 		}
1820 		if (ret_p)
1821 			break;
1822 	}
1823 	return ret_p;
1824 }
1825 
1826 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid,struct kfd_process_device ** pdd)1827 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
1828 						struct kfd_process_device **pdd)
1829 {
1830 	struct kfd_process_device *ret_p;
1831 
1832 	int idx = srcu_read_lock(&kfd_processes_srcu);
1833 
1834 	ret_p = kfd_lookup_process_device_by_pasid(pasid);
1835 	if (ret_p) {
1836 		if (pdd)
1837 			*pdd = ret_p;
1838 		kref_get(&ret_p->process->ref);
1839 
1840 		srcu_read_unlock(&kfd_processes_srcu, idx);
1841 		return ret_p->process;
1842 	}
1843 
1844 	srcu_read_unlock(&kfd_processes_srcu, idx);
1845 
1846 	if (pdd)
1847 		*pdd = NULL;
1848 
1849 	return NULL;
1850 }
1851 
1852 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1853 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1854 {
1855 	struct kfd_process *p;
1856 
1857 	int idx = srcu_read_lock(&kfd_processes_srcu);
1858 
1859 	p = find_process_by_mm(mm);
1860 	if (p)
1861 		kref_get(&p->ref);
1862 
1863 	srcu_read_unlock(&kfd_processes_srcu, idx);
1864 
1865 	return p;
1866 }
1867 
1868 /* kfd_process_evict_queues - Evict all user queues of a process
1869  *
1870  * Eviction is reference-counted per process-device. This means multiple
1871  * evictions from different sources can be nested safely.
1872  */
kfd_process_evict_queues(struct kfd_process * p,uint32_t trigger)1873 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1874 {
1875 	int r = 0;
1876 	int i;
1877 	unsigned int n_evicted = 0;
1878 
1879 	for (i = 0; i < p->n_pdds; i++) {
1880 		struct kfd_process_device *pdd = p->pdds[i];
1881 		struct device *dev = pdd->dev->adev->dev;
1882 
1883 		kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
1884 					     trigger);
1885 
1886 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1887 							    &pdd->qpd);
1888 		/* evict return -EIO if HWS is hang or asic is resetting, in this case
1889 		 * we would like to set all the queues to be in evicted state to prevent
1890 		 * them been add back since they actually not be saved right now.
1891 		 */
1892 		if (r && r != -EIO) {
1893 			dev_err(dev, "Failed to evict process queues\n");
1894 			goto fail;
1895 		}
1896 		n_evicted++;
1897 
1898 		pdd->dev->dqm->is_hws_hang = false;
1899 	}
1900 
1901 	return r;
1902 
1903 fail:
1904 	/* To keep state consistent, roll back partial eviction by
1905 	 * restoring queues
1906 	 */
1907 	for (i = 0; i < p->n_pdds; i++) {
1908 		struct kfd_process_device *pdd = p->pdds[i];
1909 
1910 		if (n_evicted == 0)
1911 			break;
1912 
1913 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1914 
1915 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1916 							      &pdd->qpd))
1917 			dev_err(pdd->dev->adev->dev,
1918 				"Failed to restore queues\n");
1919 
1920 		n_evicted--;
1921 	}
1922 
1923 	return r;
1924 }
1925 
1926 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1927 int kfd_process_restore_queues(struct kfd_process *p)
1928 {
1929 	int r, ret = 0;
1930 	int i;
1931 
1932 	for (i = 0; i < p->n_pdds; i++) {
1933 		struct kfd_process_device *pdd = p->pdds[i];
1934 		struct device *dev = pdd->dev->adev->dev;
1935 
1936 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1937 
1938 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1939 							      &pdd->qpd);
1940 		if (r) {
1941 			dev_err(dev, "Failed to restore process queues\n");
1942 			if (!ret)
1943 				ret = r;
1944 		}
1945 	}
1946 
1947 	return ret;
1948 }
1949 
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1950 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1951 {
1952 	int i;
1953 
1954 	for (i = 0; i < p->n_pdds; i++)
1955 		if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1956 			return i;
1957 	return -EINVAL;
1958 }
1959 
1960 int
kfd_process_gpuid_from_node(struct kfd_process * p,struct kfd_node * node,uint32_t * gpuid,uint32_t * gpuidx)1961 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1962 			    uint32_t *gpuid, uint32_t *gpuidx)
1963 {
1964 	int i;
1965 
1966 	for (i = 0; i < p->n_pdds; i++)
1967 		if (p->pdds[i] && p->pdds[i]->dev == node) {
1968 			*gpuid = p->pdds[i]->user_gpu_id;
1969 			*gpuidx = i;
1970 			return 0;
1971 		}
1972 	return -EINVAL;
1973 }
1974 
signal_eviction_fence(struct kfd_process * p)1975 static int signal_eviction_fence(struct kfd_process *p)
1976 {
1977 	struct dma_fence *ef;
1978 	int ret;
1979 
1980 	rcu_read_lock();
1981 	ef = dma_fence_get_rcu_safe(&p->ef);
1982 	rcu_read_unlock();
1983 	if (!ef)
1984 		return -EINVAL;
1985 
1986 	ret = dma_fence_signal(ef);
1987 	dma_fence_put(ef);
1988 
1989 	return ret;
1990 }
1991 
evict_process_worker(struct work_struct * work)1992 static void evict_process_worker(struct work_struct *work)
1993 {
1994 	int ret;
1995 	struct kfd_process *p;
1996 	struct delayed_work *dwork;
1997 
1998 	dwork = to_delayed_work(work);
1999 
2000 	/* Process termination destroys this worker thread. So during the
2001 	 * lifetime of this thread, kfd_process p will be valid
2002 	 */
2003 	p = container_of(dwork, struct kfd_process, eviction_work);
2004 
2005 	pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);
2006 	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
2007 	if (!ret) {
2008 		/* If another thread already signaled the eviction fence,
2009 		 * they are responsible stopping the queues and scheduling
2010 		 * the restore work.
2011 		 */
2012 		if (signal_eviction_fence(p) ||
2013 		    mod_delayed_work(kfd_restore_wq, &p->restore_work,
2014 				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2015 			kfd_process_restore_queues(p);
2016 
2017 		pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);
2018 	} else
2019 		pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);
2020 }
2021 
restore_process_helper(struct kfd_process * p)2022 static int restore_process_helper(struct kfd_process *p)
2023 {
2024 	int ret = 0;
2025 
2026 	/* VMs may not have been acquired yet during debugging. */
2027 	if (p->kgd_process_info) {
2028 		ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2029 			p->kgd_process_info, &p->ef);
2030 		if (ret)
2031 			return ret;
2032 	}
2033 
2034 	ret = kfd_process_restore_queues(p);
2035 	if (!ret)
2036 		pr_debug("Finished restoring process pid %d\n",
2037 			p->lead_thread->pid);
2038 	else
2039 		pr_err("Failed to restore queues of process pid %d\n",
2040 		      p->lead_thread->pid);
2041 
2042 	return ret;
2043 }
2044 
restore_process_worker(struct work_struct * work)2045 static void restore_process_worker(struct work_struct *work)
2046 {
2047 	struct delayed_work *dwork;
2048 	struct kfd_process *p;
2049 	int ret = 0;
2050 
2051 	dwork = to_delayed_work(work);
2052 
2053 	/* Process termination destroys this worker thread. So during the
2054 	 * lifetime of this thread, kfd_process p will be valid
2055 	 */
2056 	p = container_of(dwork, struct kfd_process, restore_work);
2057 	pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);
2058 
2059 	/* Setting last_restore_timestamp before successful restoration.
2060 	 * Otherwise this would have to be set by KGD (restore_process_bos)
2061 	 * before KFD BOs are unreserved. If not, the process can be evicted
2062 	 * again before the timestamp is set.
2063 	 * If restore fails, the timestamp will be set again in the next
2064 	 * attempt. This would mean that the minimum GPU quanta would be
2065 	 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2066 	 * functions)
2067 	 */
2068 
2069 	p->last_restore_timestamp = get_jiffies_64();
2070 
2071 	ret = restore_process_helper(p);
2072 	if (ret) {
2073 		pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",
2074 			 p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS);
2075 		if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
2076 				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2077 			kfd_process_restore_queues(p);
2078 	}
2079 }
2080 
kfd_suspend_all_processes(void)2081 void kfd_suspend_all_processes(void)
2082 {
2083 	struct kfd_process *p;
2084 	unsigned int temp;
2085 	int idx = srcu_read_lock(&kfd_processes_srcu);
2086 
2087 	WARN(debug_evictions, "Evicting all processes");
2088 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2089 		if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2090 			pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);
2091 		signal_eviction_fence(p);
2092 	}
2093 	srcu_read_unlock(&kfd_processes_srcu, idx);
2094 }
2095 
kfd_resume_all_processes(void)2096 int kfd_resume_all_processes(void)
2097 {
2098 	struct kfd_process *p;
2099 	unsigned int temp;
2100 	int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2101 
2102 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2103 		if (restore_process_helper(p)) {
2104 			pr_err("Restore process pid %d failed during resume\n",
2105 			      p->lead_thread->pid);
2106 			ret = -EFAULT;
2107 		}
2108 	}
2109 	srcu_read_unlock(&kfd_processes_srcu, idx);
2110 	return ret;
2111 }
2112 
kfd_reserved_mem_mmap(struct kfd_node * dev,struct kfd_process * process,struct vm_area_struct * vma)2113 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2114 			  struct vm_area_struct *vma)
2115 {
2116 	struct kfd_process_device *pdd;
2117 	struct qcm_process_device *qpd;
2118 
2119 	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2120 		dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
2121 		return -EINVAL;
2122 	}
2123 
2124 	pdd = kfd_get_process_device_data(dev, process);
2125 	if (!pdd)
2126 		return -EINVAL;
2127 	qpd = &pdd->qpd;
2128 
2129 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2130 					get_order(KFD_CWSR_TBA_TMA_SIZE));
2131 	if (!qpd->cwsr_kaddr) {
2132 		dev_err(dev->adev->dev,
2133 			"Error allocating per process CWSR buffer.\n");
2134 		return -ENOMEM;
2135 	}
2136 
2137 	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2138 		| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2139 	/* Mapping pages to user process */
2140 	return remap_pfn_range(vma, vma->vm_start,
2141 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2142 			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2143 }
2144 
2145 /* assumes caller holds process lock. */
kfd_process_drain_interrupts(struct kfd_process_device * pdd)2146 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2147 {
2148 	uint32_t irq_drain_fence[8];
2149 	uint8_t node_id = 0;
2150 	int r = 0;
2151 
2152 	if (!KFD_IS_SOC15(pdd->dev))
2153 		return 0;
2154 
2155 	pdd->process->irq_drain_is_open = true;
2156 
2157 	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2158 	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2159 							KFD_IRQ_FENCE_CLIENTID;
2160 	irq_drain_fence[3] = pdd->pasid;
2161 
2162 	/*
2163 	 * For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3]
2164 	 */
2165 	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2166 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
2167 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) {
2168 		node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2169 		irq_drain_fence[3] |= node_id << 16;
2170 	}
2171 
2172 	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2173 	if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2174 						     irq_drain_fence)) {
2175 		pdd->process->irq_drain_is_open = false;
2176 		return 0;
2177 	}
2178 
2179 	r = wait_event_interruptible(pdd->process->wait_irq_drain,
2180 				     !READ_ONCE(pdd->process->irq_drain_is_open));
2181 	if (r)
2182 		pdd->process->irq_drain_is_open = false;
2183 
2184 	return r;
2185 }
2186 
kfd_process_close_interrupt_drain(unsigned int pasid)2187 void kfd_process_close_interrupt_drain(unsigned int pasid)
2188 {
2189 	struct kfd_process *p;
2190 
2191 	p = kfd_lookup_process_by_pasid(pasid, NULL);
2192 
2193 	if (!p)
2194 		return;
2195 
2196 	WRITE_ONCE(p->irq_drain_is_open, false);
2197 	wake_up_all(&p->wait_irq_drain);
2198 	kfd_unref_process(p);
2199 }
2200 
2201 struct send_exception_work_handler_workarea {
2202 	struct work_struct work;
2203 	struct kfd_process *p;
2204 	unsigned int queue_id;
2205 	uint64_t error_reason;
2206 };
2207 
send_exception_work_handler(struct work_struct * work)2208 static void send_exception_work_handler(struct work_struct *work)
2209 {
2210 	struct send_exception_work_handler_workarea *workarea;
2211 	struct kfd_process *p;
2212 	struct queue *q;
2213 	struct mm_struct *mm;
2214 	struct kfd_context_save_area_header __user *csa_header;
2215 	uint64_t __user *err_payload_ptr;
2216 	uint64_t cur_err;
2217 	uint32_t ev_id;
2218 
2219 	workarea = container_of(work,
2220 				struct send_exception_work_handler_workarea,
2221 				work);
2222 	p = workarea->p;
2223 
2224 	mm = get_task_mm(p->lead_thread);
2225 
2226 	if (!mm)
2227 		return;
2228 
2229 	kthread_use_mm(mm);
2230 
2231 	q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2232 
2233 	if (!q)
2234 		goto out;
2235 
2236 	csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2237 
2238 	get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2239 	get_user(cur_err, err_payload_ptr);
2240 	cur_err |= workarea->error_reason;
2241 	put_user(cur_err, err_payload_ptr);
2242 	get_user(ev_id, &csa_header->err_event_id);
2243 
2244 	kfd_set_event(p, ev_id);
2245 
2246 out:
2247 	kthread_unuse_mm(mm);
2248 	mmput(mm);
2249 }
2250 
kfd_send_exception_to_runtime(struct kfd_process * p,unsigned int queue_id,uint64_t error_reason)2251 int kfd_send_exception_to_runtime(struct kfd_process *p,
2252 			unsigned int queue_id,
2253 			uint64_t error_reason)
2254 {
2255 	struct send_exception_work_handler_workarea worker;
2256 
2257 	INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2258 
2259 	worker.p = p;
2260 	worker.queue_id = queue_id;
2261 	worker.error_reason = error_reason;
2262 
2263 	schedule_work(&worker.work);
2264 	flush_work(&worker.work);
2265 	destroy_work_on_stack(&worker.work);
2266 
2267 	return 0;
2268 }
2269 
kfd_process_device_data_by_id(struct kfd_process * p,uint32_t gpu_id)2270 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2271 {
2272 	int i;
2273 
2274 	if (gpu_id) {
2275 		for (i = 0; i < p->n_pdds; i++) {
2276 			struct kfd_process_device *pdd = p->pdds[i];
2277 
2278 			if (pdd->user_gpu_id == gpu_id)
2279 				return pdd;
2280 		}
2281 	}
2282 	return NULL;
2283 }
2284 
kfd_process_get_user_gpu_id(struct kfd_process * p,uint32_t actual_gpu_id)2285 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2286 {
2287 	int i;
2288 
2289 	if (!actual_gpu_id)
2290 		return 0;
2291 
2292 	for (i = 0; i < p->n_pdds; i++) {
2293 		struct kfd_process_device *pdd = p->pdds[i];
2294 
2295 		if (pdd->dev->id == actual_gpu_id)
2296 			return pdd->user_gpu_id;
2297 	}
2298 	return -EINVAL;
2299 }
2300 
2301 #if defined(CONFIG_DEBUG_FS)
2302 
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)2303 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2304 {
2305 	struct kfd_process *p;
2306 	unsigned int temp;
2307 	int r = 0;
2308 
2309 	int idx = srcu_read_lock(&kfd_processes_srcu);
2310 
2311 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2312 		seq_printf(m, "Process %d PASID %d:\n",
2313 			   p->lead_thread->tgid, p->lead_thread->pid);
2314 
2315 		mutex_lock(&p->mutex);
2316 		r = pqm_debugfs_mqds(m, &p->pqm);
2317 		mutex_unlock(&p->mutex);
2318 
2319 		if (r)
2320 			break;
2321 	}
2322 
2323 	srcu_read_unlock(&kfd_processes_srcu, idx);
2324 
2325 	return r;
2326 }
2327 
2328 #endif
2329