1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <linux/mutex.h>
25 #include <linux/log2.h>
26 #include <linux/sched.h>
27 #include <linux/sched/mm.h>
28 #include <linux/sched/task.h>
29 #include <linux/mmu_context.h>
30 #include <linux/slab.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 #include "amdgpu_reset.h"
39
40 struct mm_struct;
41
42 #include "kfd_priv.h"
43 #include "kfd_device_queue_manager.h"
44 #include "kfd_svm.h"
45 #include "kfd_smi_events.h"
46 #include "kfd_debug.h"
47
48 /*
49 * List of struct kfd_process (field kfd_process).
50 * Unique/indexed by mm_struct*
51 */
52 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
53 DEFINE_MUTEX(kfd_processes_mutex);
54
55 DEFINE_SRCU(kfd_processes_srcu);
56
57 /* For process termination handling */
58 static struct workqueue_struct *kfd_process_wq;
59
60 /* Ordered, single-threaded workqueue for restoring evicted
61 * processes. Restoring multiple processes concurrently under memory
62 * pressure can lead to processes blocking each other from validating
63 * their BOs and result in a live-lock situation where processes
64 * remain evicted indefinitely.
65 */
66 static struct workqueue_struct *kfd_restore_wq;
67
68 static struct kfd_process *find_process(const struct task_struct *thread,
69 bool ref);
70 static void kfd_process_ref_release(struct kref *ref);
71 static struct kfd_process *create_process(const struct task_struct *thread);
72
73 static void evict_process_worker(struct work_struct *work);
74 static void restore_process_worker(struct work_struct *work);
75
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77
78 struct kfd_procfs_tree {
79 struct kobject *kobj;
80 };
81
82 static struct kfd_procfs_tree procfs;
83
84 /*
85 * Structure for SDMA activity tracking
86 */
87 struct kfd_sdma_activity_handler_workarea {
88 struct work_struct sdma_activity_work;
89 struct kfd_process_device *pdd;
90 uint64_t sdma_activity_counter;
91 };
92
93 struct temp_sdma_queue_list {
94 uint64_t __user *rptr;
95 uint64_t sdma_val;
96 unsigned int queue_id;
97 struct list_head list;
98 };
99
kfd_sdma_activity_worker(struct work_struct * work)100 static void kfd_sdma_activity_worker(struct work_struct *work)
101 {
102 struct kfd_sdma_activity_handler_workarea *workarea;
103 struct kfd_process_device *pdd;
104 uint64_t val;
105 struct mm_struct *mm;
106 struct queue *q;
107 struct qcm_process_device *qpd;
108 struct device_queue_manager *dqm;
109 int ret = 0;
110 struct temp_sdma_queue_list sdma_q_list;
111 struct temp_sdma_queue_list *sdma_q, *next;
112
113 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 sdma_activity_work);
115
116 pdd = workarea->pdd;
117 if (!pdd)
118 return;
119 dqm = pdd->dev->dqm;
120 qpd = &pdd->qpd;
121 if (!dqm || !qpd)
122 return;
123 /*
124 * Total SDMA activity is current SDMA activity + past SDMA activity
125 * Past SDMA count is stored in pdd.
126 * To get the current activity counters for all active SDMA queues,
127 * we loop over all SDMA queues and get their counts from user-space.
128 *
129 * We cannot call get_user() with dqm_lock held as it can cause
130 * a circular lock dependency situation. To read the SDMA stats,
131 * we need to do the following:
132 *
133 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 * with dqm_lock/dqm_unlock().
135 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 * Save the SDMA count for each node and also add the count to the total
137 * SDMA count counter.
138 * Its possible, during this step, a few SDMA queue nodes got deleted
139 * from the qpd->queues_list.
140 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 * If any node got deleted, its SDMA count would be captured in the sdma
142 * past activity counter. So subtract the SDMA counter stored in step 2
143 * for this node from the total SDMA count.
144 */
145 INIT_LIST_HEAD(&sdma_q_list.list);
146
147 /*
148 * Create the temp list of all SDMA queues
149 */
150 dqm_lock(dqm);
151
152 list_for_each_entry(q, &qpd->queues_list, list) {
153 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 continue;
156
157 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
158 if (!sdma_q) {
159 dqm_unlock(dqm);
160 goto cleanup;
161 }
162
163 INIT_LIST_HEAD(&sdma_q->list);
164 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
165 sdma_q->queue_id = q->properties.queue_id;
166 list_add_tail(&sdma_q->list, &sdma_q_list.list);
167 }
168
169 /*
170 * If the temp list is empty, then no SDMA queues nodes were found in
171 * qpd->queues_list. Return the past activity count as the total sdma
172 * count
173 */
174 if (list_empty(&sdma_q_list.list)) {
175 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
176 dqm_unlock(dqm);
177 return;
178 }
179
180 dqm_unlock(dqm);
181
182 /*
183 * Get the usage count for each SDMA queue in temp_list.
184 */
185 mm = get_task_mm(pdd->process->lead_thread);
186 if (!mm)
187 goto cleanup;
188
189 kthread_use_mm(mm);
190
191 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
192 val = 0;
193 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
194 if (ret) {
195 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
196 sdma_q->queue_id);
197 } else {
198 sdma_q->sdma_val = val;
199 workarea->sdma_activity_counter += val;
200 }
201 }
202
203 kthread_unuse_mm(mm);
204 mmput(mm);
205
206 /*
207 * Do a second iteration over qpd_queues_list to check if any SDMA
208 * nodes got deleted while fetching SDMA counter.
209 */
210 dqm_lock(dqm);
211
212 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
213
214 list_for_each_entry(q, &qpd->queues_list, list) {
215 if (list_empty(&sdma_q_list.list))
216 break;
217
218 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
219 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
220 continue;
221
222 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
223 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
224 (sdma_q->queue_id == q->properties.queue_id)) {
225 list_del(&sdma_q->list);
226 kfree(sdma_q);
227 break;
228 }
229 }
230 }
231
232 dqm_unlock(dqm);
233
234 /*
235 * If temp list is not empty, it implies some queues got deleted
236 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
237 * count for each node from the total SDMA count.
238 */
239 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
240 workarea->sdma_activity_counter -= sdma_q->sdma_val;
241 list_del(&sdma_q->list);
242 kfree(sdma_q);
243 }
244
245 return;
246
247 cleanup:
248 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
249 list_del(&sdma_q->list);
250 kfree(sdma_q);
251 }
252 }
253
254 /**
255 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
256 * by current process. Translates acquired wave count into number of compute units
257 * that are occupied.
258 *
259 * @attr: Handle of attribute that allows reporting of wave count. The attribute
260 * handle encapsulates GPU device it is associated with, thereby allowing collection
261 * of waves in flight, etc
262 * @buffer: Handle of user provided buffer updated with wave count
263 *
264 * Return: Number of bytes written to user buffer or an error value
265 */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)266 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
267 {
268 int cu_cnt;
269 int wave_cnt;
270 int max_waves_per_cu;
271 struct kfd_node *dev = NULL;
272 struct kfd_process *proc = NULL;
273 struct kfd_process_device *pdd = NULL;
274 int i;
275 struct kfd_cu_occupancy *cu_occupancy;
276 u32 queue_format;
277
278 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
279 dev = pdd->dev;
280 if (dev->kfd2kgd->get_cu_occupancy == NULL)
281 return -EINVAL;
282
283 cu_cnt = 0;
284 proc = pdd->process;
285 if (pdd->qpd.queue_count == 0) {
286 pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",
287 dev->id, (int)proc->lead_thread->pid);
288 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
289 }
290
291 /* Collect wave count from device if it supports */
292 wave_cnt = 0;
293 max_waves_per_cu = 0;
294
295 cu_occupancy = kcalloc(AMDGPU_MAX_QUEUES, sizeof(*cu_occupancy), GFP_KERNEL);
296 if (!cu_occupancy)
297 return -ENOMEM;
298
299 /*
300 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
301 * For AQL queues, because of cooperative dispatch we multiply the wave count
302 * by number of XCCs in the partition to get the total wave counts across all
303 * XCCs in the partition.
304 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
305 */
306 dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
307 &max_waves_per_cu, ffs(dev->xcc_mask) - 1);
308
309 for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
310 if (cu_occupancy[i].wave_cnt != 0 &&
311 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,
312 cu_occupancy[i].doorbell_off,
313 &queue_format)) {
314 if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
315 wave_cnt += cu_occupancy[i].wave_cnt;
316 else
317 wave_cnt += (NUM_XCC(dev->xcc_mask) *
318 cu_occupancy[i].wave_cnt);
319 }
320 }
321
322 /* Translate wave count to number of compute units */
323 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
324 kfree(cu_occupancy);
325 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
326 }
327
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)328 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
329 char *buffer)
330 {
331 if (strcmp(attr->name, "pasid") == 0)
332 return snprintf(buffer, PAGE_SIZE, "%d\n", 0);
333 else if (strncmp(attr->name, "vram_", 5) == 0) {
334 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
335 attr_vram);
336 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
337 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
338 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
339 attr_sdma);
340 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
341
342 INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
343 kfd_sdma_activity_worker);
344
345 sdma_activity_work_handler.pdd = pdd;
346 sdma_activity_work_handler.sdma_activity_counter = 0;
347
348 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
349
350 flush_work(&sdma_activity_work_handler.sdma_activity_work);
351 destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
352
353 return snprintf(buffer, PAGE_SIZE, "%llu\n",
354 (sdma_activity_work_handler.sdma_activity_counter)/
355 SDMA_ACTIVITY_DIVISOR);
356 } else {
357 pr_err("Invalid attribute");
358 return -EINVAL;
359 }
360
361 return 0;
362 }
363
kfd_procfs_kobj_release(struct kobject * kobj)364 static void kfd_procfs_kobj_release(struct kobject *kobj)
365 {
366 kfree(kobj);
367 }
368
369 static const struct sysfs_ops kfd_procfs_ops = {
370 .show = kfd_procfs_show,
371 };
372
373 static const struct kobj_type procfs_type = {
374 .release = kfd_procfs_kobj_release,
375 .sysfs_ops = &kfd_procfs_ops,
376 };
377
kfd_procfs_init(void)378 void kfd_procfs_init(void)
379 {
380 int ret = 0;
381
382 procfs.kobj = kfd_alloc_struct(procfs.kobj);
383 if (!procfs.kobj)
384 return;
385
386 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
387 &kfd_device->kobj, "proc");
388 if (ret) {
389 pr_warn("Could not create procfs proc folder");
390 /* If we fail to create the procfs, clean up */
391 kfd_procfs_shutdown();
392 }
393 }
394
kfd_procfs_shutdown(void)395 void kfd_procfs_shutdown(void)
396 {
397 if (procfs.kobj) {
398 kobject_del(procfs.kobj);
399 kobject_put(procfs.kobj);
400 procfs.kobj = NULL;
401 }
402 }
403
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)404 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
405 struct attribute *attr, char *buffer)
406 {
407 struct queue *q = container_of(kobj, struct queue, kobj);
408
409 if (!strcmp(attr->name, "size"))
410 return snprintf(buffer, PAGE_SIZE, "%llu",
411 q->properties.queue_size);
412 else if (!strcmp(attr->name, "type"))
413 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
414 else if (!strcmp(attr->name, "gpuid"))
415 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
416 else
417 pr_err("Invalid attribute");
418
419 return 0;
420 }
421
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)422 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
423 struct attribute *attr, char *buffer)
424 {
425 if (strcmp(attr->name, "evicted_ms") == 0) {
426 struct kfd_process_device *pdd = container_of(attr,
427 struct kfd_process_device,
428 attr_evict);
429 uint64_t evict_jiffies;
430
431 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
432
433 return snprintf(buffer,
434 PAGE_SIZE,
435 "%llu\n",
436 jiffies64_to_msecs(evict_jiffies));
437
438 /* Sysfs handle that gets CU occupancy is per device */
439 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
440 return kfd_get_cu_occupancy(attr, buffer);
441 } else {
442 pr_err("Invalid attribute");
443 }
444
445 return 0;
446 }
447
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)448 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
449 struct attribute *attr, char *buf)
450 {
451 struct kfd_process_device *pdd;
452
453 if (!strcmp(attr->name, "faults")) {
454 pdd = container_of(attr, struct kfd_process_device,
455 attr_faults);
456 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
457 }
458 if (!strcmp(attr->name, "page_in")) {
459 pdd = container_of(attr, struct kfd_process_device,
460 attr_page_in);
461 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
462 }
463 if (!strcmp(attr->name, "page_out")) {
464 pdd = container_of(attr, struct kfd_process_device,
465 attr_page_out);
466 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
467 }
468 return 0;
469 }
470
471 static struct attribute attr_queue_size = {
472 .name = "size",
473 .mode = KFD_SYSFS_FILE_MODE
474 };
475
476 static struct attribute attr_queue_type = {
477 .name = "type",
478 .mode = KFD_SYSFS_FILE_MODE
479 };
480
481 static struct attribute attr_queue_gpuid = {
482 .name = "gpuid",
483 .mode = KFD_SYSFS_FILE_MODE
484 };
485
486 static struct attribute *procfs_queue_attrs[] = {
487 &attr_queue_size,
488 &attr_queue_type,
489 &attr_queue_gpuid,
490 NULL
491 };
492 ATTRIBUTE_GROUPS(procfs_queue);
493
494 static const struct sysfs_ops procfs_queue_ops = {
495 .show = kfd_procfs_queue_show,
496 };
497
498 static const struct kobj_type procfs_queue_type = {
499 .sysfs_ops = &procfs_queue_ops,
500 .default_groups = procfs_queue_groups,
501 };
502
503 static const struct sysfs_ops procfs_stats_ops = {
504 .show = kfd_procfs_stats_show,
505 };
506
507 static const struct kobj_type procfs_stats_type = {
508 .sysfs_ops = &procfs_stats_ops,
509 .release = kfd_procfs_kobj_release,
510 };
511
512 static const struct sysfs_ops sysfs_counters_ops = {
513 .show = kfd_sysfs_counters_show,
514 };
515
516 static const struct kobj_type sysfs_counters_type = {
517 .sysfs_ops = &sysfs_counters_ops,
518 .release = kfd_procfs_kobj_release,
519 };
520
kfd_procfs_add_queue(struct queue * q)521 int kfd_procfs_add_queue(struct queue *q)
522 {
523 struct kfd_process *proc;
524 int ret;
525
526 if (!q || !q->process)
527 return -EINVAL;
528 proc = q->process;
529
530 /* Create proc/<pid>/queues/<queue id> folder */
531 if (!proc->kobj_queues)
532 return -EFAULT;
533 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
534 proc->kobj_queues, "%u", q->properties.queue_id);
535 if (ret < 0) {
536 pr_warn("Creating proc/<pid>/queues/%u failed",
537 q->properties.queue_id);
538 kobject_put(&q->kobj);
539 return ret;
540 }
541
542 return 0;
543 }
544
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)545 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
546 char *name)
547 {
548 int ret;
549
550 if (!kobj || !attr || !name)
551 return;
552
553 attr->name = name;
554 attr->mode = KFD_SYSFS_FILE_MODE;
555 sysfs_attr_init(attr);
556
557 ret = sysfs_create_file(kobj, attr);
558 if (ret)
559 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
560 }
561
kfd_procfs_add_sysfs_stats(struct kfd_process * p)562 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
563 {
564 int ret;
565 int i;
566 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
567
568 if (!p || !p->kobj)
569 return;
570
571 /*
572 * Create sysfs files for each GPU:
573 * - proc/<pid>/stats_<gpuid>/
574 * - proc/<pid>/stats_<gpuid>/evicted_ms
575 * - proc/<pid>/stats_<gpuid>/cu_occupancy
576 */
577 for (i = 0; i < p->n_pdds; i++) {
578 struct kfd_process_device *pdd = p->pdds[i];
579
580 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
581 "stats_%u", pdd->dev->id);
582 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
583 if (!pdd->kobj_stats)
584 return;
585
586 ret = kobject_init_and_add(pdd->kobj_stats,
587 &procfs_stats_type,
588 p->kobj,
589 stats_dir_filename);
590
591 if (ret) {
592 pr_warn("Creating KFD proc/stats_%s folder failed",
593 stats_dir_filename);
594 kobject_put(pdd->kobj_stats);
595 pdd->kobj_stats = NULL;
596 return;
597 }
598
599 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
600 "evicted_ms");
601 /* Add sysfs file to report compute unit occupancy */
602 if (pdd->dev->kfd2kgd->get_cu_occupancy)
603 kfd_sysfs_create_file(pdd->kobj_stats,
604 &pdd->attr_cu_occupancy,
605 "cu_occupancy");
606 }
607 }
608
kfd_procfs_add_sysfs_counters(struct kfd_process * p)609 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
610 {
611 int ret = 0;
612 int i;
613 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
614
615 if (!p || !p->kobj)
616 return;
617
618 /*
619 * Create sysfs files for each GPU which supports SVM
620 * - proc/<pid>/counters_<gpuid>/
621 * - proc/<pid>/counters_<gpuid>/faults
622 * - proc/<pid>/counters_<gpuid>/page_in
623 * - proc/<pid>/counters_<gpuid>/page_out
624 */
625 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
626 struct kfd_process_device *pdd = p->pdds[i];
627 struct kobject *kobj_counters;
628
629 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
630 "counters_%u", pdd->dev->id);
631 kobj_counters = kfd_alloc_struct(kobj_counters);
632 if (!kobj_counters)
633 return;
634
635 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
636 p->kobj, counters_dir_filename);
637 if (ret) {
638 pr_warn("Creating KFD proc/%s folder failed",
639 counters_dir_filename);
640 kobject_put(kobj_counters);
641 return;
642 }
643
644 pdd->kobj_counters = kobj_counters;
645 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
646 "faults");
647 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
648 "page_in");
649 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
650 "page_out");
651 }
652 }
653
kfd_procfs_add_sysfs_files(struct kfd_process * p)654 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
655 {
656 int i;
657
658 if (!p || !p->kobj)
659 return;
660
661 /*
662 * Create sysfs files for each GPU:
663 * - proc/<pid>/vram_<gpuid>
664 * - proc/<pid>/sdma_<gpuid>
665 */
666 for (i = 0; i < p->n_pdds; i++) {
667 struct kfd_process_device *pdd = p->pdds[i];
668
669 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
670 pdd->dev->id);
671 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
672 pdd->vram_filename);
673
674 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
675 pdd->dev->id);
676 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
677 pdd->sdma_filename);
678 }
679 }
680
kfd_procfs_del_queue(struct queue * q)681 void kfd_procfs_del_queue(struct queue *q)
682 {
683 if (!q)
684 return;
685
686 kobject_del(&q->kobj);
687 kobject_put(&q->kobj);
688 }
689
kfd_process_create_wq(void)690 int kfd_process_create_wq(void)
691 {
692 if (!kfd_process_wq)
693 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
694 if (!kfd_restore_wq)
695 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
696 WQ_FREEZABLE);
697
698 if (!kfd_process_wq || !kfd_restore_wq) {
699 kfd_process_destroy_wq();
700 return -ENOMEM;
701 }
702
703 return 0;
704 }
705
kfd_process_destroy_wq(void)706 void kfd_process_destroy_wq(void)
707 {
708 if (kfd_process_wq) {
709 destroy_workqueue(kfd_process_wq);
710 kfd_process_wq = NULL;
711 }
712 if (kfd_restore_wq) {
713 destroy_workqueue(kfd_restore_wq);
714 kfd_restore_wq = NULL;
715 }
716 }
717
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd,void ** kptr)718 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
719 struct kfd_process_device *pdd, void **kptr)
720 {
721 struct kfd_node *dev = pdd->dev;
722
723 if (kptr && *kptr) {
724 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
725 *kptr = NULL;
726 }
727
728 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
729 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
730 NULL);
731 }
732
733 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
734 * This function should be only called right after the process
735 * is created and when kfd_processes_mutex is still being held
736 * to avoid concurrency. Because of that exclusiveness, we do
737 * not need to take p->mutex.
738 */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,struct kgd_mem ** mem,void ** kptr)739 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
740 uint64_t gpu_va, uint32_t size,
741 uint32_t flags, struct kgd_mem **mem, void **kptr)
742 {
743 struct kfd_node *kdev = pdd->dev;
744 int err;
745
746 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
747 pdd->drm_priv, mem, NULL,
748 flags, false);
749 if (err)
750 goto err_alloc_mem;
751
752 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
753 pdd->drm_priv);
754 if (err)
755 goto err_map_mem;
756
757 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
758 if (err) {
759 pr_debug("Sync memory failed, wait interrupted by user signal\n");
760 goto sync_memory_failed;
761 }
762
763 if (kptr) {
764 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
765 (struct kgd_mem *)*mem, kptr, NULL);
766 if (err) {
767 pr_debug("Map GTT BO to kernel failed\n");
768 goto sync_memory_failed;
769 }
770 }
771
772 return err;
773
774 sync_memory_failed:
775 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
776
777 err_map_mem:
778 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
779 NULL);
780 err_alloc_mem:
781 *mem = NULL;
782 *kptr = NULL;
783 return err;
784 }
785
786 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
787 * process for IB usage The memory reserved is for KFD to submit
788 * IB to AMDGPU from kernel. If the memory is reserved
789 * successfully, ib_kaddr will have the CPU/kernel
790 * address. Check ib_kaddr before accessing the memory.
791 */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)792 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
793 {
794 struct qcm_process_device *qpd = &pdd->qpd;
795 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
796 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
797 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
798 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
799 struct kgd_mem *mem;
800 void *kaddr;
801 int ret;
802
803 if (qpd->ib_kaddr || !qpd->ib_base)
804 return 0;
805
806 /* ib_base is only set for dGPU */
807 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
808 &mem, &kaddr);
809 if (ret)
810 return ret;
811
812 qpd->ib_mem = mem;
813 qpd->ib_kaddr = kaddr;
814
815 return 0;
816 }
817
kfd_process_device_destroy_ib_mem(struct kfd_process_device * pdd)818 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
819 {
820 struct qcm_process_device *qpd = &pdd->qpd;
821
822 if (!qpd->ib_kaddr || !qpd->ib_base)
823 return;
824
825 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
826 }
827
kfd_create_process(struct task_struct * thread)828 struct kfd_process *kfd_create_process(struct task_struct *thread)
829 {
830 struct kfd_process *process;
831 int ret;
832
833 if (!(thread->mm && mmget_not_zero(thread->mm)))
834 return ERR_PTR(-EINVAL);
835
836 /* Only the pthreads threading model is supported. */
837 if (thread->group_leader->mm != thread->mm) {
838 mmput(thread->mm);
839 return ERR_PTR(-EINVAL);
840 }
841
842 /* If the process just called exec(3), it is possible that the
843 * cleanup of the kfd_process (following the release of the mm
844 * of the old process image) is still in the cleanup work queue.
845 * Make sure to drain any job before trying to recreate any
846 * resource for this process.
847 */
848 flush_workqueue(kfd_process_wq);
849
850 /*
851 * take kfd processes mutex before starting of process creation
852 * so there won't be a case where two threads of the same process
853 * create two kfd_process structures
854 */
855 mutex_lock(&kfd_processes_mutex);
856
857 if (kfd_is_locked()) {
858 pr_debug("KFD is locked! Cannot create process");
859 process = ERR_PTR(-EINVAL);
860 goto out;
861 }
862
863 /* A prior open of /dev/kfd could have already created the process.
864 * find_process will increase process kref in this case
865 */
866 process = find_process(thread, true);
867 if (process) {
868 pr_debug("Process already found\n");
869 } else {
870 process = create_process(thread);
871 if (IS_ERR(process))
872 goto out;
873
874 if (!procfs.kobj)
875 goto out;
876
877 process->kobj = kfd_alloc_struct(process->kobj);
878 if (!process->kobj) {
879 pr_warn("Creating procfs kobject failed");
880 goto out;
881 }
882 ret = kobject_init_and_add(process->kobj, &procfs_type,
883 procfs.kobj, "%d",
884 (int)process->lead_thread->pid);
885 if (ret) {
886 pr_warn("Creating procfs pid directory failed");
887 kobject_put(process->kobj);
888 goto out;
889 }
890
891 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
892 "pasid");
893
894 process->kobj_queues = kobject_create_and_add("queues",
895 process->kobj);
896 if (!process->kobj_queues)
897 pr_warn("Creating KFD proc/queues folder failed");
898
899 kfd_procfs_add_sysfs_stats(process);
900 kfd_procfs_add_sysfs_files(process);
901 kfd_procfs_add_sysfs_counters(process);
902
903 init_waitqueue_head(&process->wait_irq_drain);
904 }
905 out:
906 mutex_unlock(&kfd_processes_mutex);
907 mmput(thread->mm);
908
909 return process;
910 }
911
kfd_get_process(const struct task_struct * thread)912 struct kfd_process *kfd_get_process(const struct task_struct *thread)
913 {
914 struct kfd_process *process;
915
916 if (!thread->mm)
917 return ERR_PTR(-EINVAL);
918
919 /* Only the pthreads threading model is supported. */
920 if (thread->group_leader->mm != thread->mm)
921 return ERR_PTR(-EINVAL);
922
923 process = find_process(thread, false);
924 if (!process)
925 return ERR_PTR(-EINVAL);
926
927 return process;
928 }
929
find_process_by_mm(const struct mm_struct * mm)930 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
931 {
932 struct kfd_process *process;
933
934 hash_for_each_possible_rcu(kfd_processes_table, process,
935 kfd_processes, (uintptr_t)mm)
936 if (process->mm == mm)
937 return process;
938
939 return NULL;
940 }
941
find_process(const struct task_struct * thread,bool ref)942 static struct kfd_process *find_process(const struct task_struct *thread,
943 bool ref)
944 {
945 struct kfd_process *p;
946 int idx;
947
948 idx = srcu_read_lock(&kfd_processes_srcu);
949 p = find_process_by_mm(thread->mm);
950 if (p && ref)
951 kref_get(&p->ref);
952 srcu_read_unlock(&kfd_processes_srcu, idx);
953
954 return p;
955 }
956
kfd_unref_process(struct kfd_process * p)957 void kfd_unref_process(struct kfd_process *p)
958 {
959 kref_put(&p->ref, kfd_process_ref_release);
960 }
961
962 /* This increments the process->ref counter. */
kfd_lookup_process_by_pid(struct pid * pid)963 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
964 {
965 struct task_struct *task = NULL;
966 struct kfd_process *p = NULL;
967
968 if (!pid) {
969 task = current;
970 get_task_struct(task);
971 } else {
972 task = get_pid_task(pid, PIDTYPE_PID);
973 }
974
975 if (task) {
976 p = find_process(task, true);
977 put_task_struct(task);
978 }
979
980 return p;
981 }
982
kfd_process_device_free_bos(struct kfd_process_device * pdd)983 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
984 {
985 struct kfd_process *p = pdd->process;
986 void *mem;
987 int id;
988 int i;
989
990 /*
991 * Remove all handles from idr and release appropriate
992 * local memory object
993 */
994 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
995
996 for (i = 0; i < p->n_pdds; i++) {
997 struct kfd_process_device *peer_pdd = p->pdds[i];
998
999 if (!peer_pdd->drm_priv)
1000 continue;
1001 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1002 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
1003 }
1004
1005 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
1006 pdd->drm_priv, NULL);
1007 kfd_process_device_remove_obj_handle(pdd, id);
1008 }
1009 }
1010
1011 /*
1012 * Just kunmap and unpin signal BO here. It will be freed in
1013 * kfd_process_free_outstanding_kfd_bos()
1014 */
kfd_process_kunmap_signal_bo(struct kfd_process * p)1015 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1016 {
1017 struct kfd_process_device *pdd;
1018 struct kfd_node *kdev;
1019 void *mem;
1020
1021 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1022 if (!kdev)
1023 return;
1024
1025 mutex_lock(&p->mutex);
1026
1027 pdd = kfd_get_process_device_data(kdev, p);
1028 if (!pdd)
1029 goto out;
1030
1031 mem = kfd_process_device_translate_handle(
1032 pdd, GET_IDR_HANDLE(p->signal_handle));
1033 if (!mem)
1034 goto out;
1035
1036 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1037
1038 out:
1039 mutex_unlock(&p->mutex);
1040 }
1041
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)1042 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1043 {
1044 int i;
1045
1046 for (i = 0; i < p->n_pdds; i++)
1047 kfd_process_device_free_bos(p->pdds[i]);
1048 }
1049
kfd_process_destroy_pdds(struct kfd_process * p)1050 static void kfd_process_destroy_pdds(struct kfd_process *p)
1051 {
1052 int i;
1053
1054 for (i = 0; i < p->n_pdds; i++) {
1055 struct kfd_process_device *pdd = p->pdds[i];
1056
1057 pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
1058 pdd->dev->id, p->lead_thread->pid);
1059 kfd_process_device_destroy_cwsr_dgpu(pdd);
1060 kfd_process_device_destroy_ib_mem(pdd);
1061
1062 if (pdd->drm_file)
1063 fput(pdd->drm_file);
1064
1065 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1066 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1067 get_order(KFD_CWSR_TBA_TMA_SIZE));
1068
1069 idr_destroy(&pdd->alloc_idr);
1070
1071 kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1072
1073 if (pdd->dev->kfd->shared_resources.enable_mes &&
1074 pdd->proc_ctx_cpu_ptr)
1075 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
1076 &pdd->proc_ctx_bo);
1077 /*
1078 * before destroying pdd, make sure to report availability
1079 * for auto suspend
1080 */
1081 if (pdd->runtime_inuse) {
1082 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);
1083 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1084 pdd->runtime_inuse = false;
1085 }
1086
1087 kfree(pdd);
1088 p->pdds[i] = NULL;
1089 }
1090 p->n_pdds = 0;
1091 }
1092
kfd_process_remove_sysfs(struct kfd_process * p)1093 static void kfd_process_remove_sysfs(struct kfd_process *p)
1094 {
1095 struct kfd_process_device *pdd;
1096 int i;
1097
1098 if (!p->kobj)
1099 return;
1100
1101 sysfs_remove_file(p->kobj, &p->attr_pasid);
1102 kobject_del(p->kobj_queues);
1103 kobject_put(p->kobj_queues);
1104 p->kobj_queues = NULL;
1105
1106 for (i = 0; i < p->n_pdds; i++) {
1107 pdd = p->pdds[i];
1108
1109 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1110 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1111
1112 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1113 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1114 sysfs_remove_file(pdd->kobj_stats,
1115 &pdd->attr_cu_occupancy);
1116 kobject_del(pdd->kobj_stats);
1117 kobject_put(pdd->kobj_stats);
1118 pdd->kobj_stats = NULL;
1119 }
1120
1121 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1122 pdd = p->pdds[i];
1123
1124 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1125 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1126 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1127 kobject_del(pdd->kobj_counters);
1128 kobject_put(pdd->kobj_counters);
1129 pdd->kobj_counters = NULL;
1130 }
1131
1132 kobject_del(p->kobj);
1133 kobject_put(p->kobj);
1134 p->kobj = NULL;
1135 }
1136
1137 /*
1138 * If any GPU is ongoing reset, wait for reset complete.
1139 */
kfd_process_wait_gpu_reset_complete(struct kfd_process * p)1140 static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1141 {
1142 int i;
1143
1144 for (i = 0; i < p->n_pdds; i++)
1145 flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1146 }
1147
1148 /* No process locking is needed in this function, because the process
1149 * is not findable any more. We must assume that no other thread is
1150 * using it any more, otherwise we couldn't safely free the process
1151 * structure in the end.
1152 */
kfd_process_wq_release(struct work_struct * work)1153 static void kfd_process_wq_release(struct work_struct *work)
1154 {
1155 struct kfd_process *p = container_of(work, struct kfd_process,
1156 release_work);
1157 struct dma_fence *ef;
1158
1159 kfd_process_dequeue_from_all_devices(p);
1160 pqm_uninit(&p->pqm);
1161
1162 /*
1163 * If GPU in reset, user queues may still running, wait for reset complete.
1164 */
1165 kfd_process_wait_gpu_reset_complete(p);
1166
1167 /* Signal the eviction fence after user mode queues are
1168 * destroyed. This allows any BOs to be freed without
1169 * triggering pointless evictions or waiting for fences.
1170 */
1171 synchronize_rcu();
1172 ef = rcu_access_pointer(p->ef);
1173 if (ef)
1174 dma_fence_signal(ef);
1175
1176 kfd_process_remove_sysfs(p);
1177
1178 kfd_process_kunmap_signal_bo(p);
1179 kfd_process_free_outstanding_kfd_bos(p);
1180 svm_range_list_fini(p);
1181
1182 kfd_process_destroy_pdds(p);
1183 dma_fence_put(ef);
1184
1185 kfd_event_free_process(p);
1186
1187 mutex_destroy(&p->mutex);
1188
1189 put_task_struct(p->lead_thread);
1190
1191 kfree(p);
1192 }
1193
kfd_process_ref_release(struct kref * ref)1194 static void kfd_process_ref_release(struct kref *ref)
1195 {
1196 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1197
1198 INIT_WORK(&p->release_work, kfd_process_wq_release);
1199 queue_work(kfd_process_wq, &p->release_work);
1200 }
1201
kfd_process_alloc_notifier(struct mm_struct * mm)1202 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1203 {
1204 /* This increments p->ref counter if kfd process p exists */
1205 struct kfd_process *p = kfd_lookup_process_by_mm(mm);
1206
1207 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1208 }
1209
kfd_process_free_notifier(struct mmu_notifier * mn)1210 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1211 {
1212 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1213 }
1214
kfd_process_notifier_release_internal(struct kfd_process * p)1215 static void kfd_process_notifier_release_internal(struct kfd_process *p)
1216 {
1217 int i;
1218
1219 cancel_delayed_work_sync(&p->eviction_work);
1220 cancel_delayed_work_sync(&p->restore_work);
1221
1222 for (i = 0; i < p->n_pdds; i++) {
1223 struct kfd_process_device *pdd = p->pdds[i];
1224
1225 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1226 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1227 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1228 }
1229
1230 /* Indicate to other users that MM is no longer valid */
1231 p->mm = NULL;
1232 kfd_dbg_trap_disable(p);
1233
1234 if (atomic_read(&p->debugged_process_count) > 0) {
1235 struct kfd_process *target;
1236 unsigned int temp;
1237 int idx = srcu_read_lock(&kfd_processes_srcu);
1238
1239 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1240 if (target->debugger_process && target->debugger_process == p) {
1241 mutex_lock_nested(&target->mutex, 1);
1242 kfd_dbg_trap_disable(target);
1243 mutex_unlock(&target->mutex);
1244 if (atomic_read(&p->debugged_process_count) == 0)
1245 break;
1246 }
1247 }
1248
1249 srcu_read_unlock(&kfd_processes_srcu, idx);
1250 }
1251
1252 mmu_notifier_put(&p->mmu_notifier);
1253 }
1254
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1255 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1256 struct mm_struct *mm)
1257 {
1258 struct kfd_process *p;
1259
1260 /*
1261 * The kfd_process structure can not be free because the
1262 * mmu_notifier srcu is read locked
1263 */
1264 p = container_of(mn, struct kfd_process, mmu_notifier);
1265 if (WARN_ON(p->mm != mm))
1266 return;
1267
1268 mutex_lock(&kfd_processes_mutex);
1269 /*
1270 * Do early return if table is empty.
1271 *
1272 * This could potentially happen if this function is called concurrently
1273 * by mmu_notifier and by kfd_cleanup_pocesses.
1274 *
1275 */
1276 if (hash_empty(kfd_processes_table)) {
1277 mutex_unlock(&kfd_processes_mutex);
1278 return;
1279 }
1280 hash_del_rcu(&p->kfd_processes);
1281 mutex_unlock(&kfd_processes_mutex);
1282 synchronize_srcu(&kfd_processes_srcu);
1283
1284 kfd_process_notifier_release_internal(p);
1285 }
1286
1287 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1288 .release = kfd_process_notifier_release,
1289 .alloc_notifier = kfd_process_alloc_notifier,
1290 .free_notifier = kfd_process_free_notifier,
1291 };
1292
1293 /*
1294 * This code handles the case when driver is being unloaded before all
1295 * mm_struct are released. We need to safely free the kfd_process and
1296 * avoid race conditions with mmu_notifier that might try to free them.
1297 *
1298 */
kfd_cleanup_processes(void)1299 void kfd_cleanup_processes(void)
1300 {
1301 struct kfd_process *p;
1302 struct hlist_node *p_temp;
1303 unsigned int temp;
1304 HLIST_HEAD(cleanup_list);
1305
1306 /*
1307 * Move all remaining kfd_process from the process table to a
1308 * temp list for processing. Once done, callback from mmu_notifier
1309 * release will not see the kfd_process in the table and do early return,
1310 * avoiding double free issues.
1311 */
1312 mutex_lock(&kfd_processes_mutex);
1313 hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1314 hash_del_rcu(&p->kfd_processes);
1315 synchronize_srcu(&kfd_processes_srcu);
1316 hlist_add_head(&p->kfd_processes, &cleanup_list);
1317 }
1318 mutex_unlock(&kfd_processes_mutex);
1319
1320 hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1321 kfd_process_notifier_release_internal(p);
1322
1323 /*
1324 * Ensures that all outstanding free_notifier get called, triggering
1325 * the release of the kfd_process struct.
1326 */
1327 mmu_notifier_synchronize();
1328 }
1329
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1330 int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1331 {
1332 unsigned long offset;
1333 int i;
1334
1335 if (p->has_cwsr)
1336 return 0;
1337
1338 for (i = 0; i < p->n_pdds; i++) {
1339 struct kfd_node *dev = p->pdds[i]->dev;
1340 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1341
1342 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1343 continue;
1344
1345 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1346 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1347 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1348 MAP_SHARED, offset);
1349
1350 if (IS_ERR_VALUE(qpd->tba_addr)) {
1351 int err = qpd->tba_addr;
1352
1353 dev_err(dev->adev->dev,
1354 "Failure to set tba address. error %d.\n", err);
1355 qpd->tba_addr = 0;
1356 qpd->cwsr_kaddr = NULL;
1357 return err;
1358 }
1359
1360 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1361
1362 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
1363
1364 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1365 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1366 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1367 }
1368
1369 p->has_cwsr = true;
1370
1371 return 0;
1372 }
1373
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1374 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1375 {
1376 struct kfd_node *dev = pdd->dev;
1377 struct qcm_process_device *qpd = &pdd->qpd;
1378 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1379 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1380 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1381 struct kgd_mem *mem;
1382 void *kaddr;
1383 int ret;
1384
1385 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1386 return 0;
1387
1388 /* cwsr_base is only set for dGPU */
1389 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1390 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1391 if (ret)
1392 return ret;
1393
1394 qpd->cwsr_mem = mem;
1395 qpd->cwsr_kaddr = kaddr;
1396 qpd->tba_addr = qpd->cwsr_base;
1397
1398 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1399
1400 kfd_process_set_trap_debug_flag(&pdd->qpd,
1401 pdd->process->debug_trap_enabled);
1402
1403 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1404 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1405 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1406
1407 return 0;
1408 }
1409
kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device * pdd)1410 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1411 {
1412 struct kfd_node *dev = pdd->dev;
1413 struct qcm_process_device *qpd = &pdd->qpd;
1414
1415 if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1416 return;
1417
1418 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
1419 }
1420
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1421 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1422 uint64_t tba_addr,
1423 uint64_t tma_addr)
1424 {
1425 if (qpd->cwsr_kaddr) {
1426 /* KFD trap handler is bound, record as second-level TBA/TMA
1427 * in first-level TMA. First-level trap will jump to second.
1428 */
1429 uint64_t *tma =
1430 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1431 tma[0] = tba_addr;
1432 tma[1] = tma_addr;
1433 } else {
1434 /* No trap handler bound, bind as first-level TBA/TMA. */
1435 qpd->tba_addr = tba_addr;
1436 qpd->tma_addr = tma_addr;
1437 }
1438 }
1439
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1440 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1441 {
1442 int i;
1443
1444 /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1445 * boot time retry setting. Mixing processes with different
1446 * XNACK/retry settings can hang the GPU.
1447 *
1448 * Different GPUs can have different noretry settings depending
1449 * on HW bugs or limitations. We need to find at least one
1450 * XNACK mode for this process that's compatible with all GPUs.
1451 * Fortunately GPUs with retry enabled (noretry=0) can run code
1452 * built for XNACK-off. On GFXv9 it may perform slower.
1453 *
1454 * Therefore applications built for XNACK-off can always be
1455 * supported and will be our fallback if any GPU does not
1456 * support retry.
1457 */
1458 for (i = 0; i < p->n_pdds; i++) {
1459 struct kfd_node *dev = p->pdds[i]->dev;
1460
1461 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1462 * support the SVM APIs and don't need to be considered
1463 * for the XNACK mode selection.
1464 */
1465 if (!KFD_IS_SOC15(dev))
1466 continue;
1467 /* Aldebaran can always support XNACK because it can support
1468 * per-process XNACK mode selection. But let the dev->noretry
1469 * setting still influence the default XNACK mode.
1470 */
1471 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1472 if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1473 pr_debug("SRIOV platform xnack not supported\n");
1474 return false;
1475 }
1476 continue;
1477 }
1478
1479 /* GFXv10 and later GPUs do not support shader preemption
1480 * during page faults. This can lead to poor QoS for queue
1481 * management and memory-manager-related preemptions or
1482 * even deadlocks.
1483 */
1484 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1485 return false;
1486
1487 if (dev->kfd->noretry)
1488 return false;
1489 }
1490
1491 return true;
1492 }
1493
kfd_process_set_trap_debug_flag(struct qcm_process_device * qpd,bool enabled)1494 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1495 bool enabled)
1496 {
1497 if (qpd->cwsr_kaddr) {
1498 uint64_t *tma =
1499 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1500 tma[2] = enabled;
1501 }
1502 }
1503
1504 /*
1505 * On return the kfd_process is fully operational and will be freed when the
1506 * mm is released
1507 */
create_process(const struct task_struct * thread)1508 static struct kfd_process *create_process(const struct task_struct *thread)
1509 {
1510 struct kfd_process *process;
1511 struct mmu_notifier *mn;
1512 int err = -ENOMEM;
1513
1514 process = kzalloc(sizeof(*process), GFP_KERNEL);
1515 if (!process)
1516 goto err_alloc_process;
1517
1518 kref_init(&process->ref);
1519 mutex_init(&process->mutex);
1520 process->mm = thread->mm;
1521 process->lead_thread = thread->group_leader;
1522 process->n_pdds = 0;
1523 process->queues_paused = false;
1524 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1525 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1526 process->last_restore_timestamp = get_jiffies_64();
1527 err = kfd_event_init_process(process);
1528 if (err)
1529 goto err_event_init;
1530 process->is_32bit_user_mode = in_compat_syscall();
1531 process->debug_trap_enabled = false;
1532 process->debugger_process = NULL;
1533 process->exception_enable_mask = 0;
1534 atomic_set(&process->debugged_process_count, 0);
1535 sema_init(&process->runtime_enable_sema, 0);
1536
1537 err = pqm_init(&process->pqm, process);
1538 if (err != 0)
1539 goto err_process_pqm_init;
1540
1541 /* init process apertures*/
1542 err = kfd_init_apertures(process);
1543 if (err != 0)
1544 goto err_init_apertures;
1545
1546 /* Check XNACK support after PDDs are created in kfd_init_apertures */
1547 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1548
1549 err = svm_range_list_init(process);
1550 if (err)
1551 goto err_init_svm_range_list;
1552
1553 /* alloc_notifier needs to find the process in the hash table */
1554 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1555 (uintptr_t)process->mm);
1556
1557 /* Avoid free_notifier to start kfd_process_wq_release if
1558 * mmu_notifier_get failed because of pending signal.
1559 */
1560 kref_get(&process->ref);
1561
1562 /* MMU notifier registration must be the last call that can fail
1563 * because after this point we cannot unwind the process creation.
1564 * After this point, mmu_notifier_put will trigger the cleanup by
1565 * dropping the last process reference in the free_notifier.
1566 */
1567 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1568 if (IS_ERR(mn)) {
1569 err = PTR_ERR(mn);
1570 goto err_register_notifier;
1571 }
1572 BUG_ON(mn != &process->mmu_notifier);
1573
1574 kfd_unref_process(process);
1575 get_task_struct(process->lead_thread);
1576
1577 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1578
1579 return process;
1580
1581 err_register_notifier:
1582 hash_del_rcu(&process->kfd_processes);
1583 svm_range_list_fini(process);
1584 err_init_svm_range_list:
1585 kfd_process_free_outstanding_kfd_bos(process);
1586 kfd_process_destroy_pdds(process);
1587 err_init_apertures:
1588 pqm_uninit(&process->pqm);
1589 err_process_pqm_init:
1590 kfd_event_free_process(process);
1591 err_event_init:
1592 mutex_destroy(&process->mutex);
1593 kfree(process);
1594 err_alloc_process:
1595 return ERR_PTR(err);
1596 }
1597
kfd_get_process_device_data(struct kfd_node * dev,struct kfd_process * p)1598 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1599 struct kfd_process *p)
1600 {
1601 int i;
1602
1603 for (i = 0; i < p->n_pdds; i++)
1604 if (p->pdds[i]->dev == dev)
1605 return p->pdds[i];
1606
1607 return NULL;
1608 }
1609
kfd_create_process_device_data(struct kfd_node * dev,struct kfd_process * p)1610 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1611 struct kfd_process *p)
1612 {
1613 struct kfd_process_device *pdd = NULL;
1614
1615 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1616 return NULL;
1617 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1618 if (!pdd)
1619 return NULL;
1620
1621 pdd->dev = dev;
1622 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1623 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1624 pdd->qpd.dqm = dev->dqm;
1625 pdd->qpd.pqm = &p->pqm;
1626 pdd->qpd.evicted = 0;
1627 pdd->qpd.mapped_gws_queue = false;
1628 pdd->process = p;
1629 pdd->bound = PDD_UNBOUND;
1630 pdd->already_dequeued = false;
1631 pdd->runtime_inuse = false;
1632 atomic64_set(&pdd->vram_usage, 0);
1633 pdd->sdma_past_activity_counter = 0;
1634 pdd->user_gpu_id = dev->id;
1635 atomic64_set(&pdd->evict_duration_counter, 0);
1636
1637 p->pdds[p->n_pdds++] = pdd;
1638 if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1639 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1640 pdd->dev->adev,
1641 false,
1642 0);
1643
1644 /* Init idr used for memory handle translation */
1645 idr_init(&pdd->alloc_idr);
1646
1647 return pdd;
1648 }
1649
1650 /**
1651 * kfd_process_device_init_vm - Initialize a VM for a process-device
1652 *
1653 * @pdd: The process-device
1654 * @drm_file: Optional pointer to a DRM file descriptor
1655 *
1656 * If @drm_file is specified, it will be used to acquire the VM from
1657 * that file descriptor. If successful, the @pdd takes ownership of
1658 * the file descriptor.
1659 *
1660 * If @drm_file is NULL, a new VM is created.
1661 *
1662 * Returns 0 on success, -errno on failure.
1663 */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1664 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1665 struct file *drm_file)
1666 {
1667 struct amdgpu_fpriv *drv_priv;
1668 struct amdgpu_vm *avm;
1669 struct kfd_process *p;
1670 struct dma_fence *ef;
1671 struct kfd_node *dev;
1672 int ret;
1673
1674 if (!drm_file)
1675 return -EINVAL;
1676
1677 if (pdd->drm_priv)
1678 return -EBUSY;
1679
1680 ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1681 if (ret)
1682 return ret;
1683 avm = &drv_priv->vm;
1684
1685 p = pdd->process;
1686 dev = pdd->dev;
1687
1688 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1689 &p->kgd_process_info,
1690 p->ef ? NULL : &ef);
1691 if (ret) {
1692 dev_err(dev->adev->dev, "Failed to create process VM object\n");
1693 return ret;
1694 }
1695
1696 if (!p->ef)
1697 RCU_INIT_POINTER(p->ef, ef);
1698
1699 pdd->drm_priv = drm_file->private_data;
1700
1701 ret = kfd_process_device_reserve_ib_mem(pdd);
1702 if (ret)
1703 goto err_reserve_ib_mem;
1704 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1705 if (ret)
1706 goto err_init_cwsr;
1707
1708 if (unlikely(!avm->pasid)) {
1709 dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",
1710 avm);
1711 ret = -EINVAL;
1712 goto err_get_pasid;
1713 }
1714
1715 pdd->pasid = avm->pasid;
1716 pdd->drm_file = drm_file;
1717
1718 return 0;
1719
1720 err_get_pasid:
1721 kfd_process_device_destroy_cwsr_dgpu(pdd);
1722 err_init_cwsr:
1723 kfd_process_device_destroy_ib_mem(pdd);
1724 err_reserve_ib_mem:
1725 pdd->drm_priv = NULL;
1726 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1727
1728 return ret;
1729 }
1730
1731 /*
1732 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1733 * to the device.
1734 * Unbinding occurs when the process dies or the device is removed.
1735 *
1736 * Assumes that the process lock is held.
1737 */
kfd_bind_process_to_device(struct kfd_node * dev,struct kfd_process * p)1738 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1739 struct kfd_process *p)
1740 {
1741 struct kfd_process_device *pdd;
1742 int err;
1743
1744 pdd = kfd_get_process_device_data(dev, p);
1745 if (!pdd) {
1746 dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1747 return ERR_PTR(-ENOMEM);
1748 }
1749
1750 if (!pdd->drm_priv)
1751 return ERR_PTR(-ENODEV);
1752
1753 /*
1754 * signal runtime-pm system to auto resume and prevent
1755 * further runtime suspend once device pdd is created until
1756 * pdd is destroyed.
1757 */
1758 if (!pdd->runtime_inuse) {
1759 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1760 if (err < 0) {
1761 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1762 return ERR_PTR(err);
1763 }
1764 }
1765
1766 /*
1767 * make sure that runtime_usage counter is incremented just once
1768 * per pdd
1769 */
1770 pdd->runtime_inuse = true;
1771
1772 return pdd;
1773 }
1774
1775 /* Create specific handle mapped to mem from process local memory idr
1776 * Assumes that the process lock is held.
1777 */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1778 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1779 void *mem)
1780 {
1781 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1782 }
1783
1784 /* Translate specific handle from process local memory idr
1785 * Assumes that the process lock is held.
1786 */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1787 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1788 int handle)
1789 {
1790 if (handle < 0)
1791 return NULL;
1792
1793 return idr_find(&pdd->alloc_idr, handle);
1794 }
1795
1796 /* Remove specific handle from process local memory idr
1797 * Assumes that the process lock is held.
1798 */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1799 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1800 int handle)
1801 {
1802 if (handle >= 0)
1803 idr_remove(&pdd->alloc_idr, handle);
1804 }
1805
kfd_lookup_process_device_by_pasid(u32 pasid)1806 static struct kfd_process_device *kfd_lookup_process_device_by_pasid(u32 pasid)
1807 {
1808 struct kfd_process_device *ret_p = NULL;
1809 struct kfd_process *p;
1810 unsigned int temp;
1811 int i;
1812
1813 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1814 for (i = 0; i < p->n_pdds; i++) {
1815 if (p->pdds[i]->pasid == pasid) {
1816 ret_p = p->pdds[i];
1817 break;
1818 }
1819 }
1820 if (ret_p)
1821 break;
1822 }
1823 return ret_p;
1824 }
1825
1826 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid,struct kfd_process_device ** pdd)1827 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
1828 struct kfd_process_device **pdd)
1829 {
1830 struct kfd_process_device *ret_p;
1831
1832 int idx = srcu_read_lock(&kfd_processes_srcu);
1833
1834 ret_p = kfd_lookup_process_device_by_pasid(pasid);
1835 if (ret_p) {
1836 if (pdd)
1837 *pdd = ret_p;
1838 kref_get(&ret_p->process->ref);
1839
1840 srcu_read_unlock(&kfd_processes_srcu, idx);
1841 return ret_p->process;
1842 }
1843
1844 srcu_read_unlock(&kfd_processes_srcu, idx);
1845
1846 if (pdd)
1847 *pdd = NULL;
1848
1849 return NULL;
1850 }
1851
1852 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1853 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1854 {
1855 struct kfd_process *p;
1856
1857 int idx = srcu_read_lock(&kfd_processes_srcu);
1858
1859 p = find_process_by_mm(mm);
1860 if (p)
1861 kref_get(&p->ref);
1862
1863 srcu_read_unlock(&kfd_processes_srcu, idx);
1864
1865 return p;
1866 }
1867
1868 /* kfd_process_evict_queues - Evict all user queues of a process
1869 *
1870 * Eviction is reference-counted per process-device. This means multiple
1871 * evictions from different sources can be nested safely.
1872 */
kfd_process_evict_queues(struct kfd_process * p,uint32_t trigger)1873 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1874 {
1875 int r = 0;
1876 int i;
1877 unsigned int n_evicted = 0;
1878
1879 for (i = 0; i < p->n_pdds; i++) {
1880 struct kfd_process_device *pdd = p->pdds[i];
1881 struct device *dev = pdd->dev->adev->dev;
1882
1883 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
1884 trigger);
1885
1886 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1887 &pdd->qpd);
1888 /* evict return -EIO if HWS is hang or asic is resetting, in this case
1889 * we would like to set all the queues to be in evicted state to prevent
1890 * them been add back since they actually not be saved right now.
1891 */
1892 if (r && r != -EIO) {
1893 dev_err(dev, "Failed to evict process queues\n");
1894 goto fail;
1895 }
1896 n_evicted++;
1897
1898 pdd->dev->dqm->is_hws_hang = false;
1899 }
1900
1901 return r;
1902
1903 fail:
1904 /* To keep state consistent, roll back partial eviction by
1905 * restoring queues
1906 */
1907 for (i = 0; i < p->n_pdds; i++) {
1908 struct kfd_process_device *pdd = p->pdds[i];
1909
1910 if (n_evicted == 0)
1911 break;
1912
1913 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1914
1915 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1916 &pdd->qpd))
1917 dev_err(pdd->dev->adev->dev,
1918 "Failed to restore queues\n");
1919
1920 n_evicted--;
1921 }
1922
1923 return r;
1924 }
1925
1926 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1927 int kfd_process_restore_queues(struct kfd_process *p)
1928 {
1929 int r, ret = 0;
1930 int i;
1931
1932 for (i = 0; i < p->n_pdds; i++) {
1933 struct kfd_process_device *pdd = p->pdds[i];
1934 struct device *dev = pdd->dev->adev->dev;
1935
1936 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1937
1938 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1939 &pdd->qpd);
1940 if (r) {
1941 dev_err(dev, "Failed to restore process queues\n");
1942 if (!ret)
1943 ret = r;
1944 }
1945 }
1946
1947 return ret;
1948 }
1949
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1950 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1951 {
1952 int i;
1953
1954 for (i = 0; i < p->n_pdds; i++)
1955 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1956 return i;
1957 return -EINVAL;
1958 }
1959
1960 int
kfd_process_gpuid_from_node(struct kfd_process * p,struct kfd_node * node,uint32_t * gpuid,uint32_t * gpuidx)1961 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1962 uint32_t *gpuid, uint32_t *gpuidx)
1963 {
1964 int i;
1965
1966 for (i = 0; i < p->n_pdds; i++)
1967 if (p->pdds[i] && p->pdds[i]->dev == node) {
1968 *gpuid = p->pdds[i]->user_gpu_id;
1969 *gpuidx = i;
1970 return 0;
1971 }
1972 return -EINVAL;
1973 }
1974
signal_eviction_fence(struct kfd_process * p)1975 static int signal_eviction_fence(struct kfd_process *p)
1976 {
1977 struct dma_fence *ef;
1978 int ret;
1979
1980 rcu_read_lock();
1981 ef = dma_fence_get_rcu_safe(&p->ef);
1982 rcu_read_unlock();
1983 if (!ef)
1984 return -EINVAL;
1985
1986 ret = dma_fence_signal(ef);
1987 dma_fence_put(ef);
1988
1989 return ret;
1990 }
1991
evict_process_worker(struct work_struct * work)1992 static void evict_process_worker(struct work_struct *work)
1993 {
1994 int ret;
1995 struct kfd_process *p;
1996 struct delayed_work *dwork;
1997
1998 dwork = to_delayed_work(work);
1999
2000 /* Process termination destroys this worker thread. So during the
2001 * lifetime of this thread, kfd_process p will be valid
2002 */
2003 p = container_of(dwork, struct kfd_process, eviction_work);
2004
2005 pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);
2006 ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
2007 if (!ret) {
2008 /* If another thread already signaled the eviction fence,
2009 * they are responsible stopping the queues and scheduling
2010 * the restore work.
2011 */
2012 if (signal_eviction_fence(p) ||
2013 mod_delayed_work(kfd_restore_wq, &p->restore_work,
2014 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2015 kfd_process_restore_queues(p);
2016
2017 pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);
2018 } else
2019 pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);
2020 }
2021
restore_process_helper(struct kfd_process * p)2022 static int restore_process_helper(struct kfd_process *p)
2023 {
2024 int ret = 0;
2025
2026 /* VMs may not have been acquired yet during debugging. */
2027 if (p->kgd_process_info) {
2028 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2029 p->kgd_process_info, &p->ef);
2030 if (ret)
2031 return ret;
2032 }
2033
2034 ret = kfd_process_restore_queues(p);
2035 if (!ret)
2036 pr_debug("Finished restoring process pid %d\n",
2037 p->lead_thread->pid);
2038 else
2039 pr_err("Failed to restore queues of process pid %d\n",
2040 p->lead_thread->pid);
2041
2042 return ret;
2043 }
2044
restore_process_worker(struct work_struct * work)2045 static void restore_process_worker(struct work_struct *work)
2046 {
2047 struct delayed_work *dwork;
2048 struct kfd_process *p;
2049 int ret = 0;
2050
2051 dwork = to_delayed_work(work);
2052
2053 /* Process termination destroys this worker thread. So during the
2054 * lifetime of this thread, kfd_process p will be valid
2055 */
2056 p = container_of(dwork, struct kfd_process, restore_work);
2057 pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);
2058
2059 /* Setting last_restore_timestamp before successful restoration.
2060 * Otherwise this would have to be set by KGD (restore_process_bos)
2061 * before KFD BOs are unreserved. If not, the process can be evicted
2062 * again before the timestamp is set.
2063 * If restore fails, the timestamp will be set again in the next
2064 * attempt. This would mean that the minimum GPU quanta would be
2065 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2066 * functions)
2067 */
2068
2069 p->last_restore_timestamp = get_jiffies_64();
2070
2071 ret = restore_process_helper(p);
2072 if (ret) {
2073 pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",
2074 p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS);
2075 if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
2076 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2077 kfd_process_restore_queues(p);
2078 }
2079 }
2080
kfd_suspend_all_processes(void)2081 void kfd_suspend_all_processes(void)
2082 {
2083 struct kfd_process *p;
2084 unsigned int temp;
2085 int idx = srcu_read_lock(&kfd_processes_srcu);
2086
2087 WARN(debug_evictions, "Evicting all processes");
2088 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2089 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2090 pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);
2091 signal_eviction_fence(p);
2092 }
2093 srcu_read_unlock(&kfd_processes_srcu, idx);
2094 }
2095
kfd_resume_all_processes(void)2096 int kfd_resume_all_processes(void)
2097 {
2098 struct kfd_process *p;
2099 unsigned int temp;
2100 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2101
2102 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2103 if (restore_process_helper(p)) {
2104 pr_err("Restore process pid %d failed during resume\n",
2105 p->lead_thread->pid);
2106 ret = -EFAULT;
2107 }
2108 }
2109 srcu_read_unlock(&kfd_processes_srcu, idx);
2110 return ret;
2111 }
2112
kfd_reserved_mem_mmap(struct kfd_node * dev,struct kfd_process * process,struct vm_area_struct * vma)2113 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2114 struct vm_area_struct *vma)
2115 {
2116 struct kfd_process_device *pdd;
2117 struct qcm_process_device *qpd;
2118
2119 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2120 dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
2121 return -EINVAL;
2122 }
2123
2124 pdd = kfd_get_process_device_data(dev, process);
2125 if (!pdd)
2126 return -EINVAL;
2127 qpd = &pdd->qpd;
2128
2129 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2130 get_order(KFD_CWSR_TBA_TMA_SIZE));
2131 if (!qpd->cwsr_kaddr) {
2132 dev_err(dev->adev->dev,
2133 "Error allocating per process CWSR buffer.\n");
2134 return -ENOMEM;
2135 }
2136
2137 vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2138 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2139 /* Mapping pages to user process */
2140 return remap_pfn_range(vma, vma->vm_start,
2141 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2142 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2143 }
2144
2145 /* assumes caller holds process lock. */
kfd_process_drain_interrupts(struct kfd_process_device * pdd)2146 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2147 {
2148 uint32_t irq_drain_fence[8];
2149 uint8_t node_id = 0;
2150 int r = 0;
2151
2152 if (!KFD_IS_SOC15(pdd->dev))
2153 return 0;
2154
2155 pdd->process->irq_drain_is_open = true;
2156
2157 memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2158 irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2159 KFD_IRQ_FENCE_CLIENTID;
2160 irq_drain_fence[3] = pdd->pasid;
2161
2162 /*
2163 * For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3]
2164 */
2165 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2166 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
2167 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) {
2168 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2169 irq_drain_fence[3] |= node_id << 16;
2170 }
2171
2172 /* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2173 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2174 irq_drain_fence)) {
2175 pdd->process->irq_drain_is_open = false;
2176 return 0;
2177 }
2178
2179 r = wait_event_interruptible(pdd->process->wait_irq_drain,
2180 !READ_ONCE(pdd->process->irq_drain_is_open));
2181 if (r)
2182 pdd->process->irq_drain_is_open = false;
2183
2184 return r;
2185 }
2186
kfd_process_close_interrupt_drain(unsigned int pasid)2187 void kfd_process_close_interrupt_drain(unsigned int pasid)
2188 {
2189 struct kfd_process *p;
2190
2191 p = kfd_lookup_process_by_pasid(pasid, NULL);
2192
2193 if (!p)
2194 return;
2195
2196 WRITE_ONCE(p->irq_drain_is_open, false);
2197 wake_up_all(&p->wait_irq_drain);
2198 kfd_unref_process(p);
2199 }
2200
2201 struct send_exception_work_handler_workarea {
2202 struct work_struct work;
2203 struct kfd_process *p;
2204 unsigned int queue_id;
2205 uint64_t error_reason;
2206 };
2207
send_exception_work_handler(struct work_struct * work)2208 static void send_exception_work_handler(struct work_struct *work)
2209 {
2210 struct send_exception_work_handler_workarea *workarea;
2211 struct kfd_process *p;
2212 struct queue *q;
2213 struct mm_struct *mm;
2214 struct kfd_context_save_area_header __user *csa_header;
2215 uint64_t __user *err_payload_ptr;
2216 uint64_t cur_err;
2217 uint32_t ev_id;
2218
2219 workarea = container_of(work,
2220 struct send_exception_work_handler_workarea,
2221 work);
2222 p = workarea->p;
2223
2224 mm = get_task_mm(p->lead_thread);
2225
2226 if (!mm)
2227 return;
2228
2229 kthread_use_mm(mm);
2230
2231 q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2232
2233 if (!q)
2234 goto out;
2235
2236 csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2237
2238 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2239 get_user(cur_err, err_payload_ptr);
2240 cur_err |= workarea->error_reason;
2241 put_user(cur_err, err_payload_ptr);
2242 get_user(ev_id, &csa_header->err_event_id);
2243
2244 kfd_set_event(p, ev_id);
2245
2246 out:
2247 kthread_unuse_mm(mm);
2248 mmput(mm);
2249 }
2250
kfd_send_exception_to_runtime(struct kfd_process * p,unsigned int queue_id,uint64_t error_reason)2251 int kfd_send_exception_to_runtime(struct kfd_process *p,
2252 unsigned int queue_id,
2253 uint64_t error_reason)
2254 {
2255 struct send_exception_work_handler_workarea worker;
2256
2257 INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2258
2259 worker.p = p;
2260 worker.queue_id = queue_id;
2261 worker.error_reason = error_reason;
2262
2263 schedule_work(&worker.work);
2264 flush_work(&worker.work);
2265 destroy_work_on_stack(&worker.work);
2266
2267 return 0;
2268 }
2269
kfd_process_device_data_by_id(struct kfd_process * p,uint32_t gpu_id)2270 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2271 {
2272 int i;
2273
2274 if (gpu_id) {
2275 for (i = 0; i < p->n_pdds; i++) {
2276 struct kfd_process_device *pdd = p->pdds[i];
2277
2278 if (pdd->user_gpu_id == gpu_id)
2279 return pdd;
2280 }
2281 }
2282 return NULL;
2283 }
2284
kfd_process_get_user_gpu_id(struct kfd_process * p,uint32_t actual_gpu_id)2285 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2286 {
2287 int i;
2288
2289 if (!actual_gpu_id)
2290 return 0;
2291
2292 for (i = 0; i < p->n_pdds; i++) {
2293 struct kfd_process_device *pdd = p->pdds[i];
2294
2295 if (pdd->dev->id == actual_gpu_id)
2296 return pdd->user_gpu_id;
2297 }
2298 return -EINVAL;
2299 }
2300
2301 #if defined(CONFIG_DEBUG_FS)
2302
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)2303 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2304 {
2305 struct kfd_process *p;
2306 unsigned int temp;
2307 int r = 0;
2308
2309 int idx = srcu_read_lock(&kfd_processes_srcu);
2310
2311 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2312 seq_printf(m, "Process %d PASID %d:\n",
2313 p->lead_thread->tgid, p->lead_thread->pid);
2314
2315 mutex_lock(&p->mutex);
2316 r = pqm_debugfs_mqds(m, &p->pqm);
2317 mutex_unlock(&p->mutex);
2318
2319 if (r)
2320 break;
2321 }
2322
2323 srcu_read_unlock(&kfd_processes_srcu, idx);
2324
2325 return r;
2326 }
2327
2328 #endif
2329