1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2014-2018 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include <linux/dma-buf.h>
24 #include <linux/list.h>
25 #include <linux/pagemap.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <drm/ttm/ttm_tt.h>
29
30 #include <drm/drm_exec.h>
31
32 #include "amdgpu_object.h"
33 #include "amdgpu_gem.h"
34 #include "amdgpu_vm.h"
35 #include "amdgpu_hmm.h"
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu_dma_buf.h"
38 #include <uapi/linux/kfd_ioctl.h>
39 #include "amdgpu_xgmi.h"
40 #include "kfd_priv.h"
41 #include "kfd_smi_events.h"
42
43 /* Userptr restore delay, just long enough to allow consecutive VM
44 * changes to accumulate
45 */
46 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
47 #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)
48
49 /*
50 * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
51 * BO chunk
52 */
53 #define VRAM_AVAILABLITY_ALIGN (1 << 21)
54
55 /* Impose limit on how much memory KFD can use */
56 static struct {
57 uint64_t max_system_mem_limit;
58 uint64_t max_ttm_mem_limit;
59 int64_t system_mem_used;
60 int64_t ttm_mem_used;
61 spinlock_t mem_limit_lock;
62 } kfd_mem_limit;
63
64 static const char * const domain_bit_to_string[] = {
65 "CPU",
66 "GTT",
67 "VRAM",
68 "GDS",
69 "GWS",
70 "OA"
71 };
72
73 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
74
75 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
76
kfd_mem_is_attached(struct amdgpu_vm * avm,struct kgd_mem * mem)77 static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
78 struct kgd_mem *mem)
79 {
80 struct kfd_mem_attachment *entry;
81
82 list_for_each_entry(entry, &mem->attachments, list)
83 if (entry->bo_va->base.vm == avm)
84 return true;
85
86 return false;
87 }
88
89 /**
90 * reuse_dmamap() - Check whether adev can share the original
91 * userptr BO
92 *
93 * If both adev and bo_adev are in direct mapping or
94 * in the same iommu group, they can share the original BO.
95 *
96 * @adev: Device to which can or cannot share the original BO
97 * @bo_adev: Device to which allocated BO belongs to
98 *
99 * Return: returns true if adev can share original userptr BO,
100 * false otherwise.
101 */
reuse_dmamap(struct amdgpu_device * adev,struct amdgpu_device * bo_adev)102 static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
103 {
104 return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
105 (adev->dev->iommu_group == bo_adev->dev->iommu_group);
106 }
107
108 /* Set memory usage limits. Current, limits are
109 * System (TTM + userptr) memory - 15/16th System RAM
110 * TTM memory - 3/8th System RAM
111 */
amdgpu_amdkfd_gpuvm_init_mem_limits(void)112 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
113 {
114 struct sysinfo si;
115 uint64_t mem;
116
117 if (kfd_mem_limit.max_system_mem_limit)
118 return;
119
120 si_meminfo(&si);
121 mem = si.totalram - si.totalhigh;
122 mem *= si.mem_unit;
123
124 spin_lock_init(&kfd_mem_limit.mem_limit_lock);
125 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
126 if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)
127 kfd_mem_limit.max_system_mem_limit >>= 1;
128 else
129 kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;
130
131 kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;
132 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
133 (kfd_mem_limit.max_system_mem_limit >> 20),
134 (kfd_mem_limit.max_ttm_mem_limit >> 20));
135 }
136
amdgpu_amdkfd_reserve_system_mem(uint64_t size)137 void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
138 {
139 kfd_mem_limit.system_mem_used += size;
140 }
141
142 /* Estimate page table size needed to represent a given memory size
143 *
144 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
145 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
146 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize
147 * for 2MB pages for TLB efficiency. However, small allocations and
148 * fragmented system memory still need some 4KB pages. We choose a
149 * compromise that should work in most cases without reserving too
150 * much memory for page tables unnecessarily (factor 16K, >> 14).
151 */
152
153 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
154
155 /**
156 * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
157 * of buffer.
158 *
159 * @adev: Device to which allocated BO belongs to
160 * @size: Size of buffer, in bytes, encapsulated by B0. This should be
161 * equivalent to amdgpu_bo_size(BO)
162 * @alloc_flag: Flag used in allocating a BO as noted above
163 * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is
164 * managed as one compute node in driver for app
165 *
166 * Return:
167 * returns -ENOMEM in case of error, ZERO otherwise
168 */
amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device * adev,uint64_t size,u32 alloc_flag,int8_t xcp_id)169 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
170 uint64_t size, u32 alloc_flag, int8_t xcp_id)
171 {
172 uint64_t reserved_for_pt =
173 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
174 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
175 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
176 size_t system_mem_needed, ttm_mem_needed, vram_needed;
177 int ret = 0;
178 uint64_t vram_size = 0;
179
180 system_mem_needed = 0;
181 ttm_mem_needed = 0;
182 vram_needed = 0;
183 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
184 system_mem_needed = size;
185 ttm_mem_needed = size;
186 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
187 /*
188 * Conservatively round up the allocation requirement to 2 MB
189 * to avoid fragmentation caused by 4K allocations in the tail
190 * 2M BO chunk.
191 */
192 vram_needed = size;
193 /*
194 * For GFX 9.4.3, get the VRAM size from XCP structs
195 */
196 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
197 return -EINVAL;
198
199 vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
200 if (adev->apu_prefer_gtt) {
201 system_mem_needed = size;
202 ttm_mem_needed = size;
203 }
204 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
205 system_mem_needed = size;
206 } else if (!(alloc_flag &
207 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
208 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
209 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
210 return -ENOMEM;
211 }
212
213 spin_lock(&kfd_mem_limit.mem_limit_lock);
214
215 if (kfd_mem_limit.system_mem_used + system_mem_needed >
216 kfd_mem_limit.max_system_mem_limit)
217 pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
218
219 if ((kfd_mem_limit.system_mem_used + system_mem_needed >
220 kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
221 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
222 kfd_mem_limit.max_ttm_mem_limit) ||
223 (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
224 vram_size - reserved_for_pt - reserved_for_ras - atomic64_read(&adev->vram_pin_size))) {
225 ret = -ENOMEM;
226 goto release;
227 }
228
229 /* Update memory accounting by decreasing available system
230 * memory, TTM memory and GPU memory as computed above
231 */
232 WARN_ONCE(vram_needed && !adev,
233 "adev reference can't be null when vram is used");
234 if (adev && xcp_id >= 0) {
235 adev->kfd.vram_used[xcp_id] += vram_needed;
236 adev->kfd.vram_used_aligned[xcp_id] +=
237 adev->apu_prefer_gtt ?
238 vram_needed :
239 ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
240 }
241 kfd_mem_limit.system_mem_used += system_mem_needed;
242 kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
243
244 release:
245 spin_unlock(&kfd_mem_limit.mem_limit_lock);
246 return ret;
247 }
248
amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device * adev,uint64_t size,u32 alloc_flag,int8_t xcp_id)249 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
250 uint64_t size, u32 alloc_flag, int8_t xcp_id)
251 {
252 spin_lock(&kfd_mem_limit.mem_limit_lock);
253
254 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
255 kfd_mem_limit.system_mem_used -= size;
256 kfd_mem_limit.ttm_mem_used -= size;
257 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
258 WARN_ONCE(!adev,
259 "adev reference can't be null when alloc mem flags vram is set");
260 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
261 goto release;
262
263 if (adev) {
264 adev->kfd.vram_used[xcp_id] -= size;
265 if (adev->apu_prefer_gtt) {
266 adev->kfd.vram_used_aligned[xcp_id] -= size;
267 kfd_mem_limit.system_mem_used -= size;
268 kfd_mem_limit.ttm_mem_used -= size;
269 } else {
270 adev->kfd.vram_used_aligned[xcp_id] -=
271 ALIGN(size, VRAM_AVAILABLITY_ALIGN);
272 }
273 }
274 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
275 kfd_mem_limit.system_mem_used -= size;
276 } else if (!(alloc_flag &
277 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
278 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
279 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
280 goto release;
281 }
282 WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
283 "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
284 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
285 "KFD TTM memory accounting unbalanced");
286 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
287 "KFD system memory accounting unbalanced");
288
289 release:
290 spin_unlock(&kfd_mem_limit.mem_limit_lock);
291 }
292
amdgpu_amdkfd_release_notify(struct amdgpu_bo * bo)293 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
294 {
295 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
296 u32 alloc_flags = bo->kfd_bo->alloc_flags;
297 u64 size = amdgpu_bo_size(bo);
298
299 amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags,
300 bo->xcp_id);
301
302 kfree(bo->kfd_bo);
303 }
304
305 /**
306 * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
307 * about USERPTR or DOOREBELL or MMIO BO.
308 *
309 * @adev: Device for which dmamap BO is being created
310 * @mem: BO of peer device that is being DMA mapped. Provides parameters
311 * in building the dmamap BO
312 * @bo_out: Output parameter updated with handle of dmamap BO
313 */
314 static int
create_dmamap_sg_bo(struct amdgpu_device * adev,struct kgd_mem * mem,struct amdgpu_bo ** bo_out)315 create_dmamap_sg_bo(struct amdgpu_device *adev,
316 struct kgd_mem *mem, struct amdgpu_bo **bo_out)
317 {
318 struct drm_gem_object *gem_obj;
319 int ret;
320 uint64_t flags = 0;
321
322 ret = amdgpu_bo_reserve(mem->bo, false);
323 if (ret)
324 return ret;
325
326 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
327 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
328 AMDGPU_GEM_CREATE_UNCACHED);
329
330 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
331 AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
332 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);
333
334 amdgpu_bo_unreserve(mem->bo);
335
336 if (ret) {
337 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
338 return -EINVAL;
339 }
340
341 *bo_out = gem_to_amdgpu_bo(gem_obj);
342 (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
343 return ret;
344 }
345
346 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
347 * reservation object.
348 *
349 * @bo: [IN] Remove eviction fence(s) from this BO
350 * @ef: [IN] This eviction fence is removed if it
351 * is present in the shared list.
352 *
353 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
354 */
amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo * bo,struct amdgpu_amdkfd_fence * ef)355 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
356 struct amdgpu_amdkfd_fence *ef)
357 {
358 struct dma_fence *replacement;
359
360 if (!ef)
361 return -EINVAL;
362
363 /* TODO: Instead of block before we should use the fence of the page
364 * table update and TLB flush here directly.
365 */
366 replacement = dma_fence_get_stub();
367 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,
368 replacement, DMA_RESV_USAGE_BOOKKEEP);
369 dma_fence_put(replacement);
370 return 0;
371 }
372
373 /**
374 * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences
375 * @bo: the BO where to remove the evictions fences from.
376 *
377 * This functions should only be used on release when all references to the BO
378 * are already dropped. We remove the eviction fence from the private copy of
379 * the dma_resv object here since that is what is used during release to
380 * determine of the BO is idle or not.
381 */
amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo * bo)382 void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo)
383 {
384 struct dma_resv *resv = &bo->tbo.base._resv;
385 struct dma_fence *fence, *stub;
386 struct dma_resv_iter cursor;
387
388 dma_resv_assert_held(resv);
389
390 stub = dma_fence_get_stub();
391 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
392 if (!to_amdgpu_amdkfd_fence(fence))
393 continue;
394
395 dma_resv_replace_fences(resv, fence->context, stub,
396 DMA_RESV_USAGE_BOOKKEEP);
397 }
398 dma_fence_put(stub);
399 }
400
amdgpu_amdkfd_bo_validate(struct amdgpu_bo * bo,uint32_t domain,bool wait)401 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
402 bool wait)
403 {
404 struct ttm_operation_ctx ctx = { false, false };
405 int ret;
406
407 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
408 "Called with userptr BO"))
409 return -EINVAL;
410
411 /* bo has been pinned, not need validate it */
412 if (bo->tbo.pin_count)
413 return 0;
414
415 amdgpu_bo_placement_from_domain(bo, domain);
416
417 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
418 if (ret)
419 goto validate_fail;
420 if (wait)
421 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
422
423 validate_fail:
424 return ret;
425 }
426
amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo * bo,uint32_t domain,struct dma_fence * fence)427 int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
428 uint32_t domain,
429 struct dma_fence *fence)
430 {
431 int ret = amdgpu_bo_reserve(bo, false);
432
433 if (ret)
434 return ret;
435
436 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
437 if (ret)
438 goto unreserve_out;
439
440 ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1);
441 if (ret)
442 goto unreserve_out;
443
444 dma_resv_add_fence(bo->tbo.base.resv, fence,
445 DMA_RESV_USAGE_BOOKKEEP);
446
447 unreserve_out:
448 amdgpu_bo_unreserve(bo);
449
450 return ret;
451 }
452
amdgpu_amdkfd_validate_vm_bo(void * _unused,struct amdgpu_bo * bo)453 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
454 {
455 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);
456 }
457
458 /* vm_validate_pt_pd_bos - Validate page table and directory BOs
459 *
460 * Page directories are not updated here because huge page handling
461 * during page table updates can invalidate page directory entries
462 * again. Page directories are only updated after updating page
463 * tables.
464 */
vm_validate_pt_pd_bos(struct amdgpu_vm * vm,struct ww_acquire_ctx * ticket)465 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm,
466 struct ww_acquire_ctx *ticket)
467 {
468 struct amdgpu_bo *pd = vm->root.bo;
469 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
470 int ret;
471
472 ret = amdgpu_vm_validate(adev, vm, ticket,
473 amdgpu_amdkfd_validate_vm_bo, NULL);
474 if (ret) {
475 pr_err("failed to validate PT BOs\n");
476 return ret;
477 }
478
479 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);
480
481 return 0;
482 }
483
vm_update_pds(struct amdgpu_vm * vm,struct amdgpu_sync * sync)484 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
485 {
486 struct amdgpu_bo *pd = vm->root.bo;
487 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
488 int ret;
489
490 ret = amdgpu_vm_update_pdes(adev, vm, false);
491 if (ret)
492 return ret;
493
494 return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL);
495 }
496
get_pte_flags(struct amdgpu_device * adev,struct kgd_mem * mem)497 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
498 {
499 uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
500 AMDGPU_VM_MTYPE_DEFAULT;
501
502 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
503 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
504 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
505 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
506
507 return amdgpu_gem_va_map_flags(adev, mapping_flags);
508 }
509
510 /**
511 * create_sg_table() - Create an sg_table for a contiguous DMA addr range
512 * @addr: The starting address to point to
513 * @size: Size of memory area in bytes being pointed to
514 *
515 * Allocates an instance of sg_table and initializes it to point to memory
516 * area specified by input parameters. The address used to build is assumed
517 * to be DMA mapped, if needed.
518 *
519 * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
520 * because they are physically contiguous.
521 *
522 * Return: Initialized instance of SG Table or NULL
523 */
create_sg_table(uint64_t addr,uint32_t size)524 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
525 {
526 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
527
528 if (!sg)
529 return NULL;
530 if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
531 kfree(sg);
532 return NULL;
533 }
534 sg_dma_address(sg->sgl) = addr;
535 sg->sgl->length = size;
536 #ifdef CONFIG_NEED_SG_DMA_LENGTH
537 sg->sgl->dma_length = size;
538 #endif
539 return sg;
540 }
541
542 static int
kfd_mem_dmamap_userptr(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)543 kfd_mem_dmamap_userptr(struct kgd_mem *mem,
544 struct kfd_mem_attachment *attachment)
545 {
546 enum dma_data_direction direction =
547 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
548 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
549 struct ttm_operation_ctx ctx = {.interruptible = true};
550 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
551 struct amdgpu_device *adev = attachment->adev;
552 struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
553 struct ttm_tt *ttm = bo->tbo.ttm;
554 int ret;
555
556 if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
557 return -EINVAL;
558
559 ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL);
560 if (unlikely(!ttm->sg))
561 return -ENOMEM;
562
563 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */
564 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages,
565 ttm->num_pages, 0,
566 (u64)ttm->num_pages << PAGE_SHIFT,
567 GFP_KERNEL);
568 if (unlikely(ret))
569 goto free_sg;
570
571 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
572 if (unlikely(ret))
573 goto release_sg;
574
575 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
576 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
577 if (ret)
578 goto unmap_sg;
579
580 return 0;
581
582 unmap_sg:
583 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
584 release_sg:
585 pr_err("DMA map userptr failed: %d\n", ret);
586 sg_free_table(ttm->sg);
587 free_sg:
588 kfree(ttm->sg);
589 ttm->sg = NULL;
590 return ret;
591 }
592
593 static int
kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment * attachment)594 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
595 {
596 struct ttm_operation_ctx ctx = {.interruptible = true};
597 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
598
599 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
600 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
601 }
602
603 /**
604 * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
605 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
606 * @attachment: Virtual address attachment of the BO on accessing device
607 *
608 * An access request from the device that owns DOORBELL does not require DMA mapping.
609 * This is because the request doesn't go through PCIe root complex i.e. it instead
610 * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
611 *
612 * In contrast, all access requests for MMIO need to be DMA mapped without regard to
613 * device ownership. This is because access requests for MMIO go through PCIe root
614 * complex.
615 *
616 * This is accomplished in two steps:
617 * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
618 * in updating requesting device's page table
619 * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
620 * accessible. This allows an update of requesting device's page table
621 * with entries associated with DOOREBELL or MMIO memory
622 *
623 * This method is invoked in the following contexts:
624 * - Mapping of DOORBELL or MMIO BO of same or peer device
625 * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
626 *
627 * Return: ZERO if successful, NON-ZERO otherwise
628 */
629 static int
kfd_mem_dmamap_sg_bo(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)630 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
631 struct kfd_mem_attachment *attachment)
632 {
633 struct ttm_operation_ctx ctx = {.interruptible = true};
634 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
635 struct amdgpu_device *adev = attachment->adev;
636 struct ttm_tt *ttm = bo->tbo.ttm;
637 enum dma_data_direction dir;
638 dma_addr_t dma_addr;
639 bool mmio;
640 int ret;
641
642 /* Expect SG Table of dmapmap BO to be NULL */
643 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
644 if (unlikely(ttm->sg)) {
645 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
646 return -EINVAL;
647 }
648
649 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
650 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
651 dma_addr = mem->bo->tbo.sg->sgl->dma_address;
652 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
653 pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
654 dma_addr = dma_map_resource(adev->dev, dma_addr,
655 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
656 ret = dma_mapping_error(adev->dev, dma_addr);
657 if (unlikely(ret))
658 return ret;
659 pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
660
661 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
662 if (unlikely(!ttm->sg)) {
663 ret = -ENOMEM;
664 goto unmap_sg;
665 }
666
667 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
668 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
669 if (unlikely(ret))
670 goto free_sg;
671
672 return ret;
673
674 free_sg:
675 sg_free_table(ttm->sg);
676 kfree(ttm->sg);
677 ttm->sg = NULL;
678 unmap_sg:
679 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
680 dir, DMA_ATTR_SKIP_CPU_SYNC);
681 return ret;
682 }
683
684 static int
kfd_mem_dmamap_attachment(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)685 kfd_mem_dmamap_attachment(struct kgd_mem *mem,
686 struct kfd_mem_attachment *attachment)
687 {
688 switch (attachment->type) {
689 case KFD_MEM_ATT_SHARED:
690 return 0;
691 case KFD_MEM_ATT_USERPTR:
692 return kfd_mem_dmamap_userptr(mem, attachment);
693 case KFD_MEM_ATT_DMABUF:
694 return kfd_mem_dmamap_dmabuf(attachment);
695 case KFD_MEM_ATT_SG:
696 return kfd_mem_dmamap_sg_bo(mem, attachment);
697 default:
698 WARN_ON_ONCE(1);
699 }
700 return -EINVAL;
701 }
702
703 static void
kfd_mem_dmaunmap_userptr(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)704 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
705 struct kfd_mem_attachment *attachment)
706 {
707 enum dma_data_direction direction =
708 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
709 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
710 struct ttm_operation_ctx ctx = {.interruptible = false};
711 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
712 struct amdgpu_device *adev = attachment->adev;
713 struct ttm_tt *ttm = bo->tbo.ttm;
714
715 if (unlikely(!ttm->sg))
716 return;
717
718 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
719 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
720
721 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
722 sg_free_table(ttm->sg);
723 kfree(ttm->sg);
724 ttm->sg = NULL;
725 }
726
727 static void
kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment * attachment)728 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
729 {
730 /* This is a no-op. We don't want to trigger eviction fences when
731 * unmapping DMABufs. Therefore the invalidation (moving to system
732 * domain) is done in kfd_mem_dmamap_dmabuf.
733 */
734 }
735
736 /**
737 * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
738 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
739 * @attachment: Virtual address attachment of the BO on accessing device
740 *
741 * The method performs following steps:
742 * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
743 * - Free SG Table that is used to encapsulate DMA mapped memory of
744 * peer device's DOORBELL or MMIO memory
745 *
746 * This method is invoked in the following contexts:
747 * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
748 * Eviction of DOOREBELL or MMIO BO on device having access to its memory
749 *
750 * Return: void
751 */
752 static void
kfd_mem_dmaunmap_sg_bo(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)753 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
754 struct kfd_mem_attachment *attachment)
755 {
756 struct ttm_operation_ctx ctx = {.interruptible = true};
757 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
758 struct amdgpu_device *adev = attachment->adev;
759 struct ttm_tt *ttm = bo->tbo.ttm;
760 enum dma_data_direction dir;
761
762 if (unlikely(!ttm->sg)) {
763 pr_debug("SG Table of BO is NULL");
764 return;
765 }
766
767 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
768 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
769
770 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
771 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
772 dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
773 ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
774 sg_free_table(ttm->sg);
775 kfree(ttm->sg);
776 ttm->sg = NULL;
777 bo->tbo.sg = NULL;
778 }
779
780 static void
kfd_mem_dmaunmap_attachment(struct kgd_mem * mem,struct kfd_mem_attachment * attachment)781 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
782 struct kfd_mem_attachment *attachment)
783 {
784 switch (attachment->type) {
785 case KFD_MEM_ATT_SHARED:
786 break;
787 case KFD_MEM_ATT_USERPTR:
788 kfd_mem_dmaunmap_userptr(mem, attachment);
789 break;
790 case KFD_MEM_ATT_DMABUF:
791 kfd_mem_dmaunmap_dmabuf(attachment);
792 break;
793 case KFD_MEM_ATT_SG:
794 kfd_mem_dmaunmap_sg_bo(mem, attachment);
795 break;
796 default:
797 WARN_ON_ONCE(1);
798 }
799 }
800
kfd_mem_export_dmabuf(struct kgd_mem * mem)801 static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
802 {
803 if (!mem->dmabuf) {
804 struct amdgpu_device *bo_adev;
805 struct dma_buf *dmabuf;
806
807 bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
808 dmabuf = drm_gem_prime_handle_to_dmabuf(&bo_adev->ddev, bo_adev->kfd.client.file,
809 mem->gem_handle,
810 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
811 DRM_RDWR : 0);
812 if (IS_ERR(dmabuf))
813 return PTR_ERR(dmabuf);
814 mem->dmabuf = dmabuf;
815 }
816
817 return 0;
818 }
819
820 static int
kfd_mem_attach_dmabuf(struct amdgpu_device * adev,struct kgd_mem * mem,struct amdgpu_bo ** bo)821 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
822 struct amdgpu_bo **bo)
823 {
824 struct drm_gem_object *gobj;
825 int ret;
826
827 ret = kfd_mem_export_dmabuf(mem);
828 if (ret)
829 return ret;
830
831 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);
832 if (IS_ERR(gobj))
833 return PTR_ERR(gobj);
834
835 *bo = gem_to_amdgpu_bo(gobj);
836 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
837
838 return 0;
839 }
840
841 /* kfd_mem_attach - Add a BO to a VM
842 *
843 * Everything that needs to bo done only once when a BO is first added
844 * to a VM. It can later be mapped and unmapped many times without
845 * repeating these steps.
846 *
847 * 0. Create BO for DMA mapping, if needed
848 * 1. Allocate and initialize BO VA entry data structure
849 * 2. Add BO to the VM
850 * 3. Determine ASIC-specific PTE flags
851 * 4. Alloc page tables and directories if needed
852 * 4a. Validate new page tables and directories
853 */
kfd_mem_attach(struct amdgpu_device * adev,struct kgd_mem * mem,struct amdgpu_vm * vm,bool is_aql)854 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
855 struct amdgpu_vm *vm, bool is_aql)
856 {
857 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
858 unsigned long bo_size = mem->bo->tbo.base.size;
859 uint64_t va = mem->va;
860 struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
861 struct amdgpu_bo *bo[2] = {NULL, NULL};
862 struct amdgpu_bo_va *bo_va;
863 bool same_hive = false;
864 int i, ret;
865
866 if (!va) {
867 pr_err("Invalid VA when adding BO to VM\n");
868 return -EINVAL;
869 }
870
871 /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
872 *
873 * The access path of MMIO and DOORBELL BOs of is always over PCIe.
874 * In contrast the access path of VRAM BOs depens upon the type of
875 * link that connects the peer device. Access over PCIe is allowed
876 * if peer device has large BAR. In contrast, access over xGMI is
877 * allowed for both small and large BAR configurations of peer device
878 */
879 if ((adev != bo_adev && !adev->apu_prefer_gtt) &&
880 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
881 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
882 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
883 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
884 same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
885 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
886 return -EINVAL;
887 }
888
889 for (i = 0; i <= is_aql; i++) {
890 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
891 if (unlikely(!attachment[i])) {
892 ret = -ENOMEM;
893 goto unwind;
894 }
895
896 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
897 va + bo_size, vm);
898
899 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
900 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
901 (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
902 same_hive) {
903 /* Mappings on the local GPU, or VRAM mappings in the
904 * local hive, or userptr, or GTT mapping can reuse dma map
905 * address space share the original BO
906 */
907 attachment[i]->type = KFD_MEM_ATT_SHARED;
908 bo[i] = mem->bo;
909 drm_gem_object_get(&bo[i]->tbo.base);
910 } else if (i > 0) {
911 /* Multiple mappings on the same GPU share the BO */
912 attachment[i]->type = KFD_MEM_ATT_SHARED;
913 bo[i] = bo[0];
914 drm_gem_object_get(&bo[i]->tbo.base);
915 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
916 /* Create an SG BO to DMA-map userptrs on other GPUs */
917 attachment[i]->type = KFD_MEM_ATT_USERPTR;
918 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
919 if (ret)
920 goto unwind;
921 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
922 } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
923 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
924 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
925 "Handing invalid SG BO in ATTACH request");
926 attachment[i]->type = KFD_MEM_ATT_SG;
927 ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
928 if (ret)
929 goto unwind;
930 /* Enable acces to GTT and VRAM BOs of peer devices */
931 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
932 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
933 attachment[i]->type = KFD_MEM_ATT_DMABUF;
934 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
935 if (ret)
936 goto unwind;
937 pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
938 } else {
939 WARN_ONCE(true, "Handling invalid ATTACH request");
940 ret = -EINVAL;
941 goto unwind;
942 }
943
944 /* Add BO to VM internal data structures */
945 ret = amdgpu_bo_reserve(bo[i], false);
946 if (ret) {
947 pr_debug("Unable to reserve BO during memory attach");
948 goto unwind;
949 }
950 bo_va = amdgpu_vm_bo_find(vm, bo[i]);
951 if (!bo_va)
952 bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
953 else
954 ++bo_va->ref_count;
955 attachment[i]->bo_va = bo_va;
956 amdgpu_bo_unreserve(bo[i]);
957 if (unlikely(!attachment[i]->bo_va)) {
958 ret = -ENOMEM;
959 pr_err("Failed to add BO object to VM. ret == %d\n",
960 ret);
961 goto unwind;
962 }
963 attachment[i]->va = va;
964 attachment[i]->pte_flags = get_pte_flags(adev, mem);
965 attachment[i]->adev = adev;
966 list_add(&attachment[i]->list, &mem->attachments);
967
968 va += bo_size;
969 }
970
971 return 0;
972
973 unwind:
974 for (; i >= 0; i--) {
975 if (!attachment[i])
976 continue;
977 if (attachment[i]->bo_va) {
978 (void)amdgpu_bo_reserve(bo[i], true);
979 if (--attachment[i]->bo_va->ref_count == 0)
980 amdgpu_vm_bo_del(adev, attachment[i]->bo_va);
981 amdgpu_bo_unreserve(bo[i]);
982 list_del(&attachment[i]->list);
983 }
984 if (bo[i])
985 drm_gem_object_put(&bo[i]->tbo.base);
986 kfree(attachment[i]);
987 }
988 return ret;
989 }
990
kfd_mem_detach(struct kfd_mem_attachment * attachment)991 static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
992 {
993 struct amdgpu_bo *bo = attachment->bo_va->base.bo;
994
995 pr_debug("\t remove VA 0x%llx in entry %p\n",
996 attachment->va, attachment);
997 if (--attachment->bo_va->ref_count == 0)
998 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va);
999 drm_gem_object_put(&bo->tbo.base);
1000 list_del(&attachment->list);
1001 kfree(attachment);
1002 }
1003
add_kgd_mem_to_kfd_bo_list(struct kgd_mem * mem,struct amdkfd_process_info * process_info,bool userptr)1004 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1005 struct amdkfd_process_info *process_info,
1006 bool userptr)
1007 {
1008 mutex_lock(&process_info->lock);
1009 if (userptr)
1010 list_add_tail(&mem->validate_list,
1011 &process_info->userptr_valid_list);
1012 else
1013 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);
1014 mutex_unlock(&process_info->lock);
1015 }
1016
remove_kgd_mem_from_kfd_bo_list(struct kgd_mem * mem,struct amdkfd_process_info * process_info)1017 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1018 struct amdkfd_process_info *process_info)
1019 {
1020 mutex_lock(&process_info->lock);
1021 list_del(&mem->validate_list);
1022 mutex_unlock(&process_info->lock);
1023 }
1024
1025 /* Initializes user pages. It registers the MMU notifier and validates
1026 * the userptr BO in the GTT domain.
1027 *
1028 * The BO must already be on the userptr_valid_list. Otherwise an
1029 * eviction and restore may happen that leaves the new BO unmapped
1030 * with the user mode queues running.
1031 *
1032 * Takes the process_info->lock to protect against concurrent restore
1033 * workers.
1034 *
1035 * Returns 0 for success, negative errno for errors.
1036 */
init_user_pages(struct kgd_mem * mem,uint64_t user_addr,bool criu_resume)1037 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1038 bool criu_resume)
1039 {
1040 struct amdkfd_process_info *process_info = mem->process_info;
1041 struct amdgpu_bo *bo = mem->bo;
1042 struct ttm_operation_ctx ctx = { true, false };
1043 struct hmm_range *range;
1044 int ret = 0;
1045
1046 mutex_lock(&process_info->lock);
1047
1048 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);
1049 if (ret) {
1050 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
1051 goto out;
1052 }
1053
1054 ret = amdgpu_hmm_register(bo, user_addr);
1055 if (ret) {
1056 pr_err("%s: Failed to register MMU notifier: %d\n",
1057 __func__, ret);
1058 goto out;
1059 }
1060
1061 if (criu_resume) {
1062 /*
1063 * During a CRIU restore operation, the userptr buffer objects
1064 * will be validated in the restore_userptr_work worker at a
1065 * later stage when it is scheduled by another ioctl called by
1066 * CRIU master process for the target pid for restore.
1067 */
1068 mutex_lock(&process_info->notifier_lock);
1069 mem->invalid++;
1070 mutex_unlock(&process_info->notifier_lock);
1071 mutex_unlock(&process_info->lock);
1072 return 0;
1073 }
1074
1075 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);
1076 if (ret) {
1077 if (ret == -EAGAIN)
1078 pr_debug("Failed to get user pages, try again\n");
1079 else
1080 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1081 goto unregister_out;
1082 }
1083
1084 ret = amdgpu_bo_reserve(bo, true);
1085 if (ret) {
1086 pr_err("%s: Failed to reserve BO\n", __func__);
1087 goto release_out;
1088 }
1089 amdgpu_bo_placement_from_domain(bo, mem->domain);
1090 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1091 if (ret)
1092 pr_err("%s: failed to validate BO\n", __func__);
1093 amdgpu_bo_unreserve(bo);
1094
1095 release_out:
1096 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
1097 unregister_out:
1098 if (ret)
1099 amdgpu_hmm_unregister(bo);
1100 out:
1101 mutex_unlock(&process_info->lock);
1102 return ret;
1103 }
1104
1105 /* Reserving a BO and its page table BOs must happen atomically to
1106 * avoid deadlocks. Some operations update multiple VMs at once. Track
1107 * all the reservation info in a context structure. Optionally a sync
1108 * object can track VM updates.
1109 */
1110 struct bo_vm_reservation_context {
1111 /* DRM execution context for the reservation */
1112 struct drm_exec exec;
1113 /* Number of VMs reserved */
1114 unsigned int n_vms;
1115 /* Pointer to sync object */
1116 struct amdgpu_sync *sync;
1117 };
1118
1119 enum bo_vm_match {
1120 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
1121 BO_VM_MAPPED, /* Match VMs where a BO is mapped */
1122 BO_VM_ALL, /* Match all VMs a BO was added to */
1123 };
1124
1125 /**
1126 * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1127 * @mem: KFD BO structure.
1128 * @vm: the VM to reserve.
1129 * @ctx: the struct that will be used in unreserve_bo_and_vms().
1130 */
reserve_bo_and_vm(struct kgd_mem * mem,struct amdgpu_vm * vm,struct bo_vm_reservation_context * ctx)1131 static int reserve_bo_and_vm(struct kgd_mem *mem,
1132 struct amdgpu_vm *vm,
1133 struct bo_vm_reservation_context *ctx)
1134 {
1135 struct amdgpu_bo *bo = mem->bo;
1136 int ret;
1137
1138 WARN_ON(!vm);
1139
1140 ctx->n_vms = 1;
1141 ctx->sync = &mem->sync;
1142 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
1143 drm_exec_until_all_locked(&ctx->exec) {
1144 ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2);
1145 drm_exec_retry_on_contention(&ctx->exec);
1146 if (unlikely(ret))
1147 goto error;
1148
1149 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);
1150 drm_exec_retry_on_contention(&ctx->exec);
1151 if (unlikely(ret))
1152 goto error;
1153 }
1154 return 0;
1155
1156 error:
1157 pr_err("Failed to reserve buffers in ttm.\n");
1158 drm_exec_fini(&ctx->exec);
1159 return ret;
1160 }
1161
1162 /**
1163 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1164 * @mem: KFD BO structure.
1165 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1166 * is used. Otherwise, a single VM associated with the BO.
1167 * @map_type: the mapping status that will be used to filter the VMs.
1168 * @ctx: the struct that will be used in unreserve_bo_and_vms().
1169 *
1170 * Returns 0 for success, negative for failure.
1171 */
reserve_bo_and_cond_vms(struct kgd_mem * mem,struct amdgpu_vm * vm,enum bo_vm_match map_type,struct bo_vm_reservation_context * ctx)1172 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1173 struct amdgpu_vm *vm, enum bo_vm_match map_type,
1174 struct bo_vm_reservation_context *ctx)
1175 {
1176 struct kfd_mem_attachment *entry;
1177 struct amdgpu_bo *bo = mem->bo;
1178 int ret;
1179
1180 ctx->sync = &mem->sync;
1181 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
1182 DRM_EXEC_IGNORE_DUPLICATES, 0);
1183 drm_exec_until_all_locked(&ctx->exec) {
1184 ctx->n_vms = 0;
1185 list_for_each_entry(entry, &mem->attachments, list) {
1186 if ((vm && vm != entry->bo_va->base.vm) ||
1187 (entry->is_mapped != map_type
1188 && map_type != BO_VM_ALL))
1189 continue;
1190
1191 ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm,
1192 &ctx->exec, 2);
1193 drm_exec_retry_on_contention(&ctx->exec);
1194 if (unlikely(ret))
1195 goto error;
1196 ++ctx->n_vms;
1197 }
1198
1199 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);
1200 drm_exec_retry_on_contention(&ctx->exec);
1201 if (unlikely(ret))
1202 goto error;
1203 }
1204 return 0;
1205
1206 error:
1207 pr_err("Failed to reserve buffers in ttm.\n");
1208 drm_exec_fini(&ctx->exec);
1209 return ret;
1210 }
1211
1212 /**
1213 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1214 * @ctx: Reservation context to unreserve
1215 * @wait: Optionally wait for a sync object representing pending VM updates
1216 * @intr: Whether the wait is interruptible
1217 *
1218 * Also frees any resources allocated in
1219 * reserve_bo_and_(cond_)vm(s). Returns the status from
1220 * amdgpu_sync_wait.
1221 */
unreserve_bo_and_vms(struct bo_vm_reservation_context * ctx,bool wait,bool intr)1222 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1223 bool wait, bool intr)
1224 {
1225 int ret = 0;
1226
1227 if (wait)
1228 ret = amdgpu_sync_wait(ctx->sync, intr);
1229
1230 drm_exec_fini(&ctx->exec);
1231 ctx->sync = NULL;
1232 return ret;
1233 }
1234
unmap_bo_from_gpuvm(struct kgd_mem * mem,struct kfd_mem_attachment * entry,struct amdgpu_sync * sync)1235 static int unmap_bo_from_gpuvm(struct kgd_mem *mem,
1236 struct kfd_mem_attachment *entry,
1237 struct amdgpu_sync *sync)
1238 {
1239 struct amdgpu_bo_va *bo_va = entry->bo_va;
1240 struct amdgpu_device *adev = entry->adev;
1241 struct amdgpu_vm *vm = bo_va->base.vm;
1242
1243 if (bo_va->queue_refcount) {
1244 pr_debug("bo_va->queue_refcount %d\n", bo_va->queue_refcount);
1245 return -EBUSY;
1246 }
1247
1248 (void)amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
1249
1250 (void)amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
1251
1252 (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
1253
1254 return 0;
1255 }
1256
update_gpuvm_pte(struct kgd_mem * mem,struct kfd_mem_attachment * entry,struct amdgpu_sync * sync)1257 static int update_gpuvm_pte(struct kgd_mem *mem,
1258 struct kfd_mem_attachment *entry,
1259 struct amdgpu_sync *sync)
1260 {
1261 struct amdgpu_bo_va *bo_va = entry->bo_va;
1262 struct amdgpu_device *adev = entry->adev;
1263 int ret;
1264
1265 ret = kfd_mem_dmamap_attachment(mem, entry);
1266 if (ret)
1267 return ret;
1268
1269 /* Update the page tables */
1270 ret = amdgpu_vm_bo_update(adev, bo_va, false);
1271 if (ret) {
1272 pr_err("amdgpu_vm_bo_update failed\n");
1273 return ret;
1274 }
1275
1276 return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
1277 }
1278
map_bo_to_gpuvm(struct kgd_mem * mem,struct kfd_mem_attachment * entry,struct amdgpu_sync * sync,bool no_update_pte)1279 static int map_bo_to_gpuvm(struct kgd_mem *mem,
1280 struct kfd_mem_attachment *entry,
1281 struct amdgpu_sync *sync,
1282 bool no_update_pte)
1283 {
1284 int ret;
1285
1286 /* Set virtual address for the allocation */
1287 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0,
1288 amdgpu_bo_size(entry->bo_va->base.bo),
1289 entry->pte_flags);
1290 if (ret) {
1291 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1292 entry->va, ret);
1293 return ret;
1294 }
1295
1296 if (no_update_pte)
1297 return 0;
1298
1299 ret = update_gpuvm_pte(mem, entry, sync);
1300 if (ret) {
1301 pr_err("update_gpuvm_pte() failed\n");
1302 goto update_gpuvm_pte_failed;
1303 }
1304
1305 return 0;
1306
1307 update_gpuvm_pte_failed:
1308 unmap_bo_from_gpuvm(mem, entry, sync);
1309 kfd_mem_dmaunmap_attachment(mem, entry);
1310 return ret;
1311 }
1312
process_validate_vms(struct amdkfd_process_info * process_info,struct ww_acquire_ctx * ticket)1313 static int process_validate_vms(struct amdkfd_process_info *process_info,
1314 struct ww_acquire_ctx *ticket)
1315 {
1316 struct amdgpu_vm *peer_vm;
1317 int ret;
1318
1319 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1320 vm_list_node) {
1321 ret = vm_validate_pt_pd_bos(peer_vm, ticket);
1322 if (ret)
1323 return ret;
1324 }
1325
1326 return 0;
1327 }
1328
process_sync_pds_resv(struct amdkfd_process_info * process_info,struct amdgpu_sync * sync)1329 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1330 struct amdgpu_sync *sync)
1331 {
1332 struct amdgpu_vm *peer_vm;
1333 int ret;
1334
1335 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1336 vm_list_node) {
1337 struct amdgpu_bo *pd = peer_vm->root.bo;
1338
1339 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,
1340 AMDGPU_SYNC_NE_OWNER,
1341 AMDGPU_FENCE_OWNER_KFD);
1342 if (ret)
1343 return ret;
1344 }
1345
1346 return 0;
1347 }
1348
process_update_pds(struct amdkfd_process_info * process_info,struct amdgpu_sync * sync)1349 static int process_update_pds(struct amdkfd_process_info *process_info,
1350 struct amdgpu_sync *sync)
1351 {
1352 struct amdgpu_vm *peer_vm;
1353 int ret;
1354
1355 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1356 vm_list_node) {
1357 ret = vm_update_pds(peer_vm, sync);
1358 if (ret)
1359 return ret;
1360 }
1361
1362 return 0;
1363 }
1364
init_kfd_vm(struct amdgpu_vm * vm,void ** process_info,struct dma_fence ** ef)1365 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1366 struct dma_fence **ef)
1367 {
1368 struct amdkfd_process_info *info = NULL;
1369 int ret;
1370
1371 if (!*process_info) {
1372 info = kzalloc(sizeof(*info), GFP_KERNEL);
1373 if (!info)
1374 return -ENOMEM;
1375
1376 mutex_init(&info->lock);
1377 mutex_init(&info->notifier_lock);
1378 INIT_LIST_HEAD(&info->vm_list_head);
1379 INIT_LIST_HEAD(&info->kfd_bo_list);
1380 INIT_LIST_HEAD(&info->userptr_valid_list);
1381 INIT_LIST_HEAD(&info->userptr_inval_list);
1382
1383 info->eviction_fence =
1384 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
1385 current->mm,
1386 NULL);
1387 if (!info->eviction_fence) {
1388 pr_err("Failed to create eviction fence\n");
1389 ret = -ENOMEM;
1390 goto create_evict_fence_fail;
1391 }
1392
1393 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
1394 INIT_DELAYED_WORK(&info->restore_userptr_work,
1395 amdgpu_amdkfd_restore_userptr_worker);
1396
1397 *process_info = info;
1398 }
1399
1400 vm->process_info = *process_info;
1401
1402 /* Validate page directory and attach eviction fence */
1403 ret = amdgpu_bo_reserve(vm->root.bo, true);
1404 if (ret)
1405 goto reserve_pd_fail;
1406 ret = vm_validate_pt_pd_bos(vm, NULL);
1407 if (ret) {
1408 pr_err("validate_pt_pd_bos() failed\n");
1409 goto validate_pd_fail;
1410 }
1411 ret = amdgpu_bo_sync_wait(vm->root.bo,
1412 AMDGPU_FENCE_OWNER_KFD, false);
1413 if (ret)
1414 goto wait_pd_fail;
1415 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
1416 if (ret)
1417 goto reserve_shared_fail;
1418 dma_resv_add_fence(vm->root.bo->tbo.base.resv,
1419 &vm->process_info->eviction_fence->base,
1420 DMA_RESV_USAGE_BOOKKEEP);
1421 amdgpu_bo_unreserve(vm->root.bo);
1422
1423 /* Update process info */
1424 mutex_lock(&vm->process_info->lock);
1425 list_add_tail(&vm->vm_list_node,
1426 &(vm->process_info->vm_list_head));
1427 vm->process_info->n_vms++;
1428 if (ef)
1429 *ef = dma_fence_get(&vm->process_info->eviction_fence->base);
1430 mutex_unlock(&vm->process_info->lock);
1431
1432 return 0;
1433
1434 reserve_shared_fail:
1435 wait_pd_fail:
1436 validate_pd_fail:
1437 amdgpu_bo_unreserve(vm->root.bo);
1438 reserve_pd_fail:
1439 vm->process_info = NULL;
1440 if (info) {
1441 dma_fence_put(&info->eviction_fence->base);
1442 *process_info = NULL;
1443 put_pid(info->pid);
1444 create_evict_fence_fail:
1445 mutex_destroy(&info->lock);
1446 mutex_destroy(&info->notifier_lock);
1447 kfree(info);
1448 }
1449 return ret;
1450 }
1451
1452 /**
1453 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1454 * @bo: Handle of buffer object being pinned
1455 * @domain: Domain into which BO should be pinned
1456 *
1457 * - USERPTR BOs are UNPINNABLE and will return error
1458 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1459 * PIN count incremented. It is valid to PIN a BO multiple times
1460 *
1461 * Return: ZERO if successful in pinning, Non-Zero in case of error.
1462 */
amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo * bo,u32 domain)1463 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1464 {
1465 int ret = 0;
1466
1467 ret = amdgpu_bo_reserve(bo, false);
1468 if (unlikely(ret))
1469 return ret;
1470
1471 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {
1472 /*
1473 * If bo is not contiguous on VRAM, move to system memory first to ensure
1474 * we can get contiguous VRAM space after evicting other BOs.
1475 */
1476 if (!(bo->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) {
1477 struct ttm_operation_ctx ctx = { true, false };
1478
1479 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
1480 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1481 if (unlikely(ret)) {
1482 pr_debug("validate bo 0x%p to GTT failed %d\n", &bo->tbo, ret);
1483 goto out;
1484 }
1485 }
1486 }
1487
1488 ret = amdgpu_bo_pin(bo, domain);
1489 if (ret)
1490 pr_err("Error in Pinning BO to domain: %d\n", domain);
1491
1492 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
1493 out:
1494 amdgpu_bo_unreserve(bo);
1495 return ret;
1496 }
1497
1498 /**
1499 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1500 * @bo: Handle of buffer object being unpinned
1501 *
1502 * - Is a illegal request for USERPTR BOs and is ignored
1503 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1504 * PIN count decremented. Calls to UNPIN must balance calls to PIN
1505 */
amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo * bo)1506 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1507 {
1508 int ret = 0;
1509
1510 ret = amdgpu_bo_reserve(bo, false);
1511 if (unlikely(ret))
1512 return;
1513
1514 amdgpu_bo_unpin(bo);
1515 amdgpu_bo_unreserve(bo);
1516 }
1517
amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device * adev,struct amdgpu_vm * avm,void ** process_info,struct dma_fence ** ef)1518 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1519 struct amdgpu_vm *avm,
1520 void **process_info,
1521 struct dma_fence **ef)
1522 {
1523 int ret;
1524
1525 /* Already a compute VM? */
1526 if (avm->process_info)
1527 return -EINVAL;
1528
1529 /* Convert VM into a compute VM */
1530 ret = amdgpu_vm_make_compute(adev, avm);
1531 if (ret)
1532 return ret;
1533
1534 /* Initialize KFD part of the VM and process info */
1535 ret = init_kfd_vm(avm, process_info, ef);
1536 if (ret)
1537 return ret;
1538
1539 amdgpu_vm_set_task_info(avm);
1540
1541 return 0;
1542 }
1543
amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device * adev,struct amdgpu_vm * vm)1544 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1545 struct amdgpu_vm *vm)
1546 {
1547 struct amdkfd_process_info *process_info = vm->process_info;
1548
1549 if (!process_info)
1550 return;
1551
1552 /* Update process info */
1553 mutex_lock(&process_info->lock);
1554 process_info->n_vms--;
1555 list_del(&vm->vm_list_node);
1556 mutex_unlock(&process_info->lock);
1557
1558 vm->process_info = NULL;
1559
1560 /* Release per-process resources when last compute VM is destroyed */
1561 if (!process_info->n_vms) {
1562 WARN_ON(!list_empty(&process_info->kfd_bo_list));
1563 WARN_ON(!list_empty(&process_info->userptr_valid_list));
1564 WARN_ON(!list_empty(&process_info->userptr_inval_list));
1565
1566 dma_fence_put(&process_info->eviction_fence->base);
1567 cancel_delayed_work_sync(&process_info->restore_userptr_work);
1568 put_pid(process_info->pid);
1569 mutex_destroy(&process_info->lock);
1570 mutex_destroy(&process_info->notifier_lock);
1571 kfree(process_info);
1572 }
1573 }
1574
amdgpu_amdkfd_gpuvm_get_process_page_dir(void * drm_priv)1575 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1576 {
1577 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1578 struct amdgpu_bo *pd = avm->root.bo;
1579 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1580
1581 if (adev->asic_type < CHIP_VEGA10)
1582 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1583 return avm->pd_phys_addr;
1584 }
1585
amdgpu_amdkfd_block_mmu_notifications(void * p)1586 void amdgpu_amdkfd_block_mmu_notifications(void *p)
1587 {
1588 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1589
1590 mutex_lock(&pinfo->lock);
1591 WRITE_ONCE(pinfo->block_mmu_notifications, true);
1592 mutex_unlock(&pinfo->lock);
1593 }
1594
amdgpu_amdkfd_criu_resume(void * p)1595 int amdgpu_amdkfd_criu_resume(void *p)
1596 {
1597 int ret = 0;
1598 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1599
1600 mutex_lock(&pinfo->lock);
1601 pr_debug("scheduling work\n");
1602 mutex_lock(&pinfo->notifier_lock);
1603 pinfo->evicted_bos++;
1604 mutex_unlock(&pinfo->notifier_lock);
1605 if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1606 ret = -EINVAL;
1607 goto out_unlock;
1608 }
1609 WRITE_ONCE(pinfo->block_mmu_notifications, false);
1610 queue_delayed_work(system_freezable_wq,
1611 &pinfo->restore_userptr_work, 0);
1612
1613 out_unlock:
1614 mutex_unlock(&pinfo->lock);
1615 return ret;
1616 }
1617
amdgpu_amdkfd_get_available_memory(struct amdgpu_device * adev,uint8_t xcp_id)1618 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
1619 uint8_t xcp_id)
1620 {
1621 uint64_t reserved_for_pt =
1622 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1623 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1624 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
1625 ssize_t available;
1626 uint64_t vram_available, system_mem_available, ttm_mem_available;
1627
1628 spin_lock(&kfd_mem_limit.mem_limit_lock);
1629 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1630 - adev->kfd.vram_used_aligned[xcp_id]
1631 - atomic64_read(&adev->vram_pin_size)
1632 - reserved_for_pt
1633 - reserved_for_ras;
1634
1635 if (adev->apu_prefer_gtt) {
1636 system_mem_available = no_system_mem_limit ?
1637 kfd_mem_limit.max_system_mem_limit :
1638 kfd_mem_limit.max_system_mem_limit -
1639 kfd_mem_limit.system_mem_used;
1640
1641 ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
1642 kfd_mem_limit.ttm_mem_used;
1643
1644 available = min3(system_mem_available, ttm_mem_available,
1645 vram_available);
1646 available = ALIGN_DOWN(available, PAGE_SIZE);
1647 } else {
1648 available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
1649 }
1650
1651 spin_unlock(&kfd_mem_limit.mem_limit_lock);
1652
1653 if (available < 0)
1654 available = 0;
1655
1656 return available;
1657 }
1658
amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(struct amdgpu_device * adev,uint64_t va,uint64_t size,void * drm_priv,struct kgd_mem ** mem,uint64_t * offset,uint32_t flags,bool criu_resume)1659 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1660 struct amdgpu_device *adev, uint64_t va, uint64_t size,
1661 void *drm_priv, struct kgd_mem **mem,
1662 uint64_t *offset, uint32_t flags, bool criu_resume)
1663 {
1664 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1665 struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
1666 enum ttm_bo_type bo_type = ttm_bo_type_device;
1667 struct sg_table *sg = NULL;
1668 uint64_t user_addr = 0;
1669 struct amdgpu_bo *bo;
1670 struct drm_gem_object *gobj = NULL;
1671 u32 domain, alloc_domain;
1672 uint64_t aligned_size;
1673 int8_t xcp_id = -1;
1674 u64 alloc_flags;
1675 int ret;
1676
1677 /*
1678 * Check on which domain to allocate BO
1679 */
1680 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1681 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1682
1683 if (adev->apu_prefer_gtt) {
1684 domain = AMDGPU_GEM_DOMAIN_GTT;
1685 alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1686 alloc_flags = 0;
1687 } else {
1688 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1689 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1690 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1691
1692 /* For contiguous VRAM allocation */
1693 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS)
1694 alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1695 }
1696 xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
1697 0 : fpriv->xcp_id;
1698 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1699 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1700 alloc_flags = 0;
1701 } else {
1702 domain = AMDGPU_GEM_DOMAIN_GTT;
1703 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1704 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1705
1706 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1707 if (!offset || !*offset)
1708 return -EINVAL;
1709 user_addr = untagged_addr(*offset);
1710 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1711 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1712 bo_type = ttm_bo_type_sg;
1713 if (size > UINT_MAX)
1714 return -EINVAL;
1715 sg = create_sg_table(*offset, size);
1716 if (!sg)
1717 return -ENOMEM;
1718 } else {
1719 return -EINVAL;
1720 }
1721 }
1722
1723 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
1724 alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;
1725 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)
1726 alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;
1727 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)
1728 alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;
1729
1730 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1731 if (!*mem) {
1732 ret = -ENOMEM;
1733 goto err;
1734 }
1735 INIT_LIST_HEAD(&(*mem)->attachments);
1736 mutex_init(&(*mem)->lock);
1737 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1738
1739 /* Workaround for AQL queue wraparound bug. Map the same
1740 * memory twice. That means we only actually allocate half
1741 * the memory.
1742 */
1743 if ((*mem)->aql_queue)
1744 size >>= 1;
1745 aligned_size = PAGE_ALIGN(size);
1746
1747 (*mem)->alloc_flags = flags;
1748
1749 amdgpu_sync_create(&(*mem)->sync);
1750
1751 ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags,
1752 xcp_id);
1753 if (ret) {
1754 pr_debug("Insufficient memory\n");
1755 goto err_reserve_limit;
1756 }
1757
1758 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
1759 va, (*mem)->aql_queue ? size << 1 : size,
1760 domain_string(alloc_domain), xcp_id);
1761
1762 ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
1763 bo_type, NULL, &gobj, xcp_id + 1);
1764 if (ret) {
1765 pr_debug("Failed to create BO on domain %s. ret %d\n",
1766 domain_string(alloc_domain), ret);
1767 goto err_bo_create;
1768 }
1769 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
1770 if (ret) {
1771 pr_debug("Failed to allow vma node access. ret %d\n", ret);
1772 goto err_node_allow;
1773 }
1774 ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle);
1775 if (ret)
1776 goto err_gem_handle_create;
1777 bo = gem_to_amdgpu_bo(gobj);
1778 if (bo_type == ttm_bo_type_sg) {
1779 bo->tbo.sg = sg;
1780 bo->tbo.ttm->sg = sg;
1781 }
1782 bo->kfd_bo = *mem;
1783 (*mem)->bo = bo;
1784 if (user_addr)
1785 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1786
1787 (*mem)->va = va;
1788 (*mem)->domain = domain;
1789 (*mem)->mapped_to_gpu_memory = 0;
1790 (*mem)->process_info = avm->process_info;
1791
1792 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1793
1794 if (user_addr) {
1795 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);
1796 ret = init_user_pages(*mem, user_addr, criu_resume);
1797 if (ret)
1798 goto allocate_init_user_pages_failed;
1799 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1800 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1801 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1802 if (ret) {
1803 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1804 goto err_pin_bo;
1805 }
1806 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1807 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1808 } else {
1809 mutex_lock(&avm->process_info->lock);
1810 if (avm->process_info->eviction_fence &&
1811 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
1812 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
1813 &avm->process_info->eviction_fence->base);
1814 mutex_unlock(&avm->process_info->lock);
1815 if (ret)
1816 goto err_validate_bo;
1817 }
1818
1819 if (offset)
1820 *offset = amdgpu_bo_mmap_offset(bo);
1821
1822 return 0;
1823
1824 allocate_init_user_pages_failed:
1825 err_pin_bo:
1826 err_validate_bo:
1827 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1828 drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle);
1829 err_gem_handle_create:
1830 drm_vma_node_revoke(&gobj->vma_node, drm_priv);
1831 err_node_allow:
1832 /* Don't unreserve system mem limit twice */
1833 goto err_reserve_limit;
1834 err_bo_create:
1835 amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);
1836 err_reserve_limit:
1837 amdgpu_sync_free(&(*mem)->sync);
1838 mutex_destroy(&(*mem)->lock);
1839 if (gobj)
1840 drm_gem_object_put(gobj);
1841 else
1842 kfree(*mem);
1843 err:
1844 if (sg) {
1845 sg_free_table(sg);
1846 kfree(sg);
1847 }
1848 return ret;
1849 }
1850
amdgpu_amdkfd_gpuvm_free_memory_of_gpu(struct amdgpu_device * adev,struct kgd_mem * mem,void * drm_priv,uint64_t * size)1851 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1852 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1853 uint64_t *size)
1854 {
1855 struct amdkfd_process_info *process_info = mem->process_info;
1856 unsigned long bo_size = mem->bo->tbo.base.size;
1857 bool use_release_notifier = (mem->bo->kfd_bo == mem);
1858 struct kfd_mem_attachment *entry, *tmp;
1859 struct bo_vm_reservation_context ctx;
1860 unsigned int mapped_to_gpu_memory;
1861 int ret;
1862 bool is_imported = false;
1863
1864 mutex_lock(&mem->lock);
1865
1866 /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1867 if (mem->alloc_flags &
1868 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1869 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1870 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
1871 }
1872
1873 mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1874 is_imported = mem->is_imported;
1875 mutex_unlock(&mem->lock);
1876 /* lock is not needed after this, since mem is unused and will
1877 * be freed anyway
1878 */
1879
1880 if (mapped_to_gpu_memory > 0) {
1881 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1882 mem->va, bo_size);
1883 return -EBUSY;
1884 }
1885
1886 /* Make sure restore workers don't access the BO any more */
1887 mutex_lock(&process_info->lock);
1888 list_del(&mem->validate_list);
1889 mutex_unlock(&process_info->lock);
1890
1891 /* Cleanup user pages and MMU notifiers */
1892 if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
1893 amdgpu_hmm_unregister(mem->bo);
1894 mutex_lock(&process_info->notifier_lock);
1895 amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range);
1896 mutex_unlock(&process_info->notifier_lock);
1897 }
1898
1899 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1900 if (unlikely(ret))
1901 return ret;
1902
1903 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1904 process_info->eviction_fence);
1905 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1906 mem->va + bo_size * (1 + mem->aql_queue));
1907
1908 /* Remove from VM internal data structures */
1909 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1910 kfd_mem_dmaunmap_attachment(mem, entry);
1911 kfd_mem_detach(entry);
1912 }
1913
1914 ret = unreserve_bo_and_vms(&ctx, false, false);
1915
1916 /* Free the sync object */
1917 amdgpu_sync_free(&mem->sync);
1918
1919 /* If the SG is not NULL, it's one we created for a doorbell or mmio
1920 * remap BO. We need to free it.
1921 */
1922 if (mem->bo->tbo.sg) {
1923 sg_free_table(mem->bo->tbo.sg);
1924 kfree(mem->bo->tbo.sg);
1925 }
1926
1927 /* Update the size of the BO being freed if it was allocated from
1928 * VRAM and is not imported. For APP APU VRAM allocations are done
1929 * in GTT domain
1930 */
1931 if (size) {
1932 if (!is_imported &&
1933 (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
1934 (adev->apu_prefer_gtt &&
1935 mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
1936 *size = bo_size;
1937 else
1938 *size = 0;
1939 }
1940
1941 /* Free the BO*/
1942 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
1943 drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle);
1944 if (mem->dmabuf) {
1945 dma_buf_put(mem->dmabuf);
1946 mem->dmabuf = NULL;
1947 }
1948 mutex_destroy(&mem->lock);
1949
1950 /* If this releases the last reference, it will end up calling
1951 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1952 * this needs to be the last call here.
1953 */
1954 drm_gem_object_put(&mem->bo->tbo.base);
1955
1956 /*
1957 * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1958 * explicitly free it here.
1959 */
1960 if (!use_release_notifier)
1961 kfree(mem);
1962
1963 return ret;
1964 }
1965
amdgpu_amdkfd_gpuvm_map_memory_to_gpu(struct amdgpu_device * adev,struct kgd_mem * mem,void * drm_priv)1966 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1967 struct amdgpu_device *adev, struct kgd_mem *mem,
1968 void *drm_priv)
1969 {
1970 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1971 int ret;
1972 struct amdgpu_bo *bo;
1973 uint32_t domain;
1974 struct kfd_mem_attachment *entry;
1975 struct bo_vm_reservation_context ctx;
1976 unsigned long bo_size;
1977 bool is_invalid_userptr = false;
1978
1979 bo = mem->bo;
1980 if (!bo) {
1981 pr_err("Invalid BO when mapping memory to GPU\n");
1982 return -EINVAL;
1983 }
1984
1985 /* Make sure restore is not running concurrently. Since we
1986 * don't map invalid userptr BOs, we rely on the next restore
1987 * worker to do the mapping
1988 */
1989 mutex_lock(&mem->process_info->lock);
1990
1991 /* Lock notifier lock. If we find an invalid userptr BO, we can be
1992 * sure that the MMU notifier is no longer running
1993 * concurrently and the queues are actually stopped
1994 */
1995 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1996 mutex_lock(&mem->process_info->notifier_lock);
1997 is_invalid_userptr = !!mem->invalid;
1998 mutex_unlock(&mem->process_info->notifier_lock);
1999 }
2000
2001 mutex_lock(&mem->lock);
2002
2003 domain = mem->domain;
2004 bo_size = bo->tbo.base.size;
2005
2006 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
2007 mem->va,
2008 mem->va + bo_size * (1 + mem->aql_queue),
2009 avm, domain_string(domain));
2010
2011 if (!kfd_mem_is_attached(avm, mem)) {
2012 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue);
2013 if (ret)
2014 goto out;
2015 }
2016
2017 ret = reserve_bo_and_vm(mem, avm, &ctx);
2018 if (unlikely(ret))
2019 goto out;
2020
2021 /* Userptr can be marked as "not invalid", but not actually be
2022 * validated yet (still in the system domain). In that case
2023 * the queues are still stopped and we can leave mapping for
2024 * the next restore worker
2025 */
2026 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
2027 bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
2028 is_invalid_userptr = true;
2029
2030 ret = vm_validate_pt_pd_bos(avm, NULL);
2031 if (unlikely(ret))
2032 goto out_unreserve;
2033
2034 list_for_each_entry(entry, &mem->attachments, list) {
2035 if (entry->bo_va->base.vm != avm || entry->is_mapped)
2036 continue;
2037
2038 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
2039 entry->va, entry->va + bo_size, entry);
2040
2041 ret = map_bo_to_gpuvm(mem, entry, ctx.sync,
2042 is_invalid_userptr);
2043 if (ret) {
2044 pr_err("Failed to map bo to gpuvm\n");
2045 goto out_unreserve;
2046 }
2047
2048 ret = vm_update_pds(avm, ctx.sync);
2049 if (ret) {
2050 pr_err("Failed to update page directories\n");
2051 goto out_unreserve;
2052 }
2053
2054 entry->is_mapped = true;
2055 mem->mapped_to_gpu_memory++;
2056 pr_debug("\t INC mapping count %d\n",
2057 mem->mapped_to_gpu_memory);
2058 }
2059
2060 ret = unreserve_bo_and_vms(&ctx, false, false);
2061
2062 goto out;
2063
2064 out_unreserve:
2065 unreserve_bo_and_vms(&ctx, false, false);
2066 out:
2067 mutex_unlock(&mem->process_info->lock);
2068 mutex_unlock(&mem->lock);
2069 return ret;
2070 }
2071
amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem * mem,void * drm_priv)2072 int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2073 {
2074 struct kfd_mem_attachment *entry;
2075 struct amdgpu_vm *vm;
2076 int ret;
2077
2078 vm = drm_priv_to_vm(drm_priv);
2079
2080 mutex_lock(&mem->lock);
2081
2082 ret = amdgpu_bo_reserve(mem->bo, true);
2083 if (ret)
2084 goto out;
2085
2086 list_for_each_entry(entry, &mem->attachments, list) {
2087 if (entry->bo_va->base.vm != vm)
2088 continue;
2089 if (entry->bo_va->base.bo->tbo.ttm &&
2090 !entry->bo_va->base.bo->tbo.ttm->sg)
2091 continue;
2092
2093 kfd_mem_dmaunmap_attachment(mem, entry);
2094 }
2095
2096 amdgpu_bo_unreserve(mem->bo);
2097 out:
2098 mutex_unlock(&mem->lock);
2099
2100 return ret;
2101 }
2102
amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(struct amdgpu_device * adev,struct kgd_mem * mem,void * drm_priv)2103 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2104 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2105 {
2106 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2107 unsigned long bo_size = mem->bo->tbo.base.size;
2108 struct kfd_mem_attachment *entry;
2109 struct bo_vm_reservation_context ctx;
2110 int ret;
2111
2112 mutex_lock(&mem->lock);
2113
2114 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
2115 if (unlikely(ret))
2116 goto out;
2117 /* If no VMs were reserved, it means the BO wasn't actually mapped */
2118 if (ctx.n_vms == 0) {
2119 ret = -EINVAL;
2120 goto unreserve_out;
2121 }
2122
2123 ret = vm_validate_pt_pd_bos(avm, NULL);
2124 if (unlikely(ret))
2125 goto unreserve_out;
2126
2127 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2128 mem->va,
2129 mem->va + bo_size * (1 + mem->aql_queue),
2130 avm);
2131
2132 list_for_each_entry(entry, &mem->attachments, list) {
2133 if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2134 continue;
2135
2136 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2137 entry->va, entry->va + bo_size, entry);
2138
2139 ret = unmap_bo_from_gpuvm(mem, entry, ctx.sync);
2140 if (ret)
2141 goto unreserve_out;
2142
2143 entry->is_mapped = false;
2144
2145 mem->mapped_to_gpu_memory--;
2146 pr_debug("\t DEC mapping count %d\n",
2147 mem->mapped_to_gpu_memory);
2148 }
2149
2150 unreserve_out:
2151 unreserve_bo_and_vms(&ctx, false, false);
2152 out:
2153 mutex_unlock(&mem->lock);
2154 return ret;
2155 }
2156
amdgpu_amdkfd_gpuvm_sync_memory(struct amdgpu_device * adev,struct kgd_mem * mem,bool intr)2157 int amdgpu_amdkfd_gpuvm_sync_memory(
2158 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2159 {
2160 struct amdgpu_sync sync;
2161 int ret;
2162
2163 amdgpu_sync_create(&sync);
2164
2165 mutex_lock(&mem->lock);
2166 amdgpu_sync_clone(&mem->sync, &sync);
2167 mutex_unlock(&mem->lock);
2168
2169 ret = amdgpu_sync_wait(&sync, intr);
2170 amdgpu_sync_free(&sync);
2171 return ret;
2172 }
2173
2174 /**
2175 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2176 * @bo: Buffer object to be mapped
2177 * @bo_gart: Return bo reference
2178 *
2179 * Before return, bo reference count is incremented. To release the reference and unpin/
2180 * unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2181 */
amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo * bo,struct amdgpu_bo ** bo_gart)2182 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart)
2183 {
2184 int ret;
2185
2186 ret = amdgpu_bo_reserve(bo, true);
2187 if (ret) {
2188 pr_err("Failed to reserve bo. ret %d\n", ret);
2189 goto err_reserve_bo_failed;
2190 }
2191
2192 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2193 if (ret) {
2194 pr_err("Failed to pin bo. ret %d\n", ret);
2195 goto err_pin_bo_failed;
2196 }
2197
2198 ret = amdgpu_ttm_alloc_gart(&bo->tbo);
2199 if (ret) {
2200 pr_err("Failed to bind bo to GART. ret %d\n", ret);
2201 goto err_map_bo_gart_failed;
2202 }
2203
2204 amdgpu_amdkfd_remove_eviction_fence(
2205 bo, bo->vm_bo->vm->process_info->eviction_fence);
2206
2207 amdgpu_bo_unreserve(bo);
2208
2209 *bo_gart = amdgpu_bo_ref(bo);
2210
2211 return 0;
2212
2213 err_map_bo_gart_failed:
2214 amdgpu_bo_unpin(bo);
2215 err_pin_bo_failed:
2216 amdgpu_bo_unreserve(bo);
2217 err_reserve_bo_failed:
2218
2219 return ret;
2220 }
2221
2222 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2223 *
2224 * @mem: Buffer object to be mapped for CPU access
2225 * @kptr[out]: pointer in kernel CPU address space
2226 * @size[out]: size of the buffer
2227 *
2228 * Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2229 * from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2230 * validate_list, so the GPU mapping can be restored after a page table was
2231 * evicted.
2232 *
2233 * Return: 0 on success, error code on failure
2234 */
amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem * mem,void ** kptr,uint64_t * size)2235 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2236 void **kptr, uint64_t *size)
2237 {
2238 int ret;
2239 struct amdgpu_bo *bo = mem->bo;
2240
2241 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
2242 pr_err("userptr can't be mapped to kernel\n");
2243 return -EINVAL;
2244 }
2245
2246 mutex_lock(&mem->process_info->lock);
2247
2248 ret = amdgpu_bo_reserve(bo, true);
2249 if (ret) {
2250 pr_err("Failed to reserve bo. ret %d\n", ret);
2251 goto bo_reserve_failed;
2252 }
2253
2254 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2255 if (ret) {
2256 pr_err("Failed to pin bo. ret %d\n", ret);
2257 goto pin_failed;
2258 }
2259
2260 ret = amdgpu_bo_kmap(bo, kptr);
2261 if (ret) {
2262 pr_err("Failed to map bo to kernel. ret %d\n", ret);
2263 goto kmap_failed;
2264 }
2265
2266 amdgpu_amdkfd_remove_eviction_fence(
2267 bo, mem->process_info->eviction_fence);
2268
2269 if (size)
2270 *size = amdgpu_bo_size(bo);
2271
2272 amdgpu_bo_unreserve(bo);
2273
2274 mutex_unlock(&mem->process_info->lock);
2275 return 0;
2276
2277 kmap_failed:
2278 amdgpu_bo_unpin(bo);
2279 pin_failed:
2280 amdgpu_bo_unreserve(bo);
2281 bo_reserve_failed:
2282 mutex_unlock(&mem->process_info->lock);
2283
2284 return ret;
2285 }
2286
2287 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2288 *
2289 * @mem: Buffer object to be unmapped for CPU access
2290 *
2291 * Removes the kernel CPU mapping and unpins the BO. It does not restore the
2292 * eviction fence, so this function should only be used for cleanup before the
2293 * BO is destroyed.
2294 */
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem * mem)2295 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2296 {
2297 struct amdgpu_bo *bo = mem->bo;
2298
2299 (void)amdgpu_bo_reserve(bo, true);
2300 amdgpu_bo_kunmap(bo);
2301 amdgpu_bo_unpin(bo);
2302 amdgpu_bo_unreserve(bo);
2303 }
2304
amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device * adev,struct kfd_vm_fault_info * mem)2305 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2306 struct kfd_vm_fault_info *mem)
2307 {
2308 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
2309 *mem = *adev->gmc.vm_fault_info;
2310 mb(); /* make sure read happened */
2311 atomic_set(&adev->gmc.vm_fault_info_updated, 0);
2312 }
2313 return 0;
2314 }
2315
import_obj_create(struct amdgpu_device * adev,struct dma_buf * dma_buf,struct drm_gem_object * obj,uint64_t va,void * drm_priv,struct kgd_mem ** mem,uint64_t * size,uint64_t * mmap_offset)2316 static int import_obj_create(struct amdgpu_device *adev,
2317 struct dma_buf *dma_buf,
2318 struct drm_gem_object *obj,
2319 uint64_t va, void *drm_priv,
2320 struct kgd_mem **mem, uint64_t *size,
2321 uint64_t *mmap_offset)
2322 {
2323 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2324 struct amdgpu_bo *bo;
2325 int ret;
2326
2327 bo = gem_to_amdgpu_bo(obj);
2328 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2329 AMDGPU_GEM_DOMAIN_GTT)))
2330 /* Only VRAM and GTT BOs are supported */
2331 return -EINVAL;
2332
2333 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2334 if (!*mem)
2335 return -ENOMEM;
2336
2337 ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
2338 if (ret)
2339 goto err_free_mem;
2340
2341 if (size)
2342 *size = amdgpu_bo_size(bo);
2343
2344 if (mmap_offset)
2345 *mmap_offset = amdgpu_bo_mmap_offset(bo);
2346
2347 INIT_LIST_HEAD(&(*mem)->attachments);
2348 mutex_init(&(*mem)->lock);
2349
2350 (*mem)->alloc_flags =
2351 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2352 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2353 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2354 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2355
2356 get_dma_buf(dma_buf);
2357 (*mem)->dmabuf = dma_buf;
2358 (*mem)->bo = bo;
2359 (*mem)->va = va;
2360 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) &&
2361 !adev->apu_prefer_gtt ?
2362 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2363
2364 (*mem)->mapped_to_gpu_memory = 0;
2365 (*mem)->process_info = avm->process_info;
2366 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
2367 amdgpu_sync_create(&(*mem)->sync);
2368 (*mem)->is_imported = true;
2369
2370 mutex_lock(&avm->process_info->lock);
2371 if (avm->process_info->eviction_fence &&
2372 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
2373 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain,
2374 &avm->process_info->eviction_fence->base);
2375 mutex_unlock(&avm->process_info->lock);
2376 if (ret)
2377 goto err_remove_mem;
2378
2379 return 0;
2380
2381 err_remove_mem:
2382 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
2383 drm_vma_node_revoke(&obj->vma_node, drm_priv);
2384 err_free_mem:
2385 kfree(*mem);
2386 return ret;
2387 }
2388
amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device * adev,int fd,uint64_t va,void * drm_priv,struct kgd_mem ** mem,uint64_t * size,uint64_t * mmap_offset)2389 int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd,
2390 uint64_t va, void *drm_priv,
2391 struct kgd_mem **mem, uint64_t *size,
2392 uint64_t *mmap_offset)
2393 {
2394 struct drm_gem_object *obj;
2395 uint32_t handle;
2396 int ret;
2397
2398 ret = drm_gem_prime_fd_to_handle(&adev->ddev, adev->kfd.client.file, fd,
2399 &handle);
2400 if (ret)
2401 return ret;
2402 obj = drm_gem_object_lookup(adev->kfd.client.file, handle);
2403 if (!obj) {
2404 ret = -EINVAL;
2405 goto err_release_handle;
2406 }
2407
2408 ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size,
2409 mmap_offset);
2410 if (ret)
2411 goto err_put_obj;
2412
2413 (*mem)->gem_handle = handle;
2414
2415 return 0;
2416
2417 err_put_obj:
2418 drm_gem_object_put(obj);
2419 err_release_handle:
2420 drm_gem_handle_delete(adev->kfd.client.file, handle);
2421 return ret;
2422 }
2423
amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem * mem,struct dma_buf ** dma_buf)2424 int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2425 struct dma_buf **dma_buf)
2426 {
2427 int ret;
2428
2429 mutex_lock(&mem->lock);
2430 ret = kfd_mem_export_dmabuf(mem);
2431 if (ret)
2432 goto out;
2433
2434 get_dma_buf(mem->dmabuf);
2435 *dma_buf = mem->dmabuf;
2436 out:
2437 mutex_unlock(&mem->lock);
2438 return ret;
2439 }
2440
2441 /* Evict a userptr BO by stopping the queues if necessary
2442 *
2443 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2444 * cannot do any memory allocations, and cannot take any locks that
2445 * are held elsewhere while allocating memory.
2446 *
2447 * It doesn't do anything to the BO itself. The real work happens in
2448 * restore, where we get updated page addresses. This function only
2449 * ensures that GPU access to the BO is stopped.
2450 */
amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier * mni,unsigned long cur_seq,struct kgd_mem * mem)2451 int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
2452 unsigned long cur_seq, struct kgd_mem *mem)
2453 {
2454 struct amdkfd_process_info *process_info = mem->process_info;
2455 int r = 0;
2456
2457 /* Do not process MMU notifications during CRIU restore until
2458 * KFD_CRIU_OP_RESUME IOCTL is received
2459 */
2460 if (READ_ONCE(process_info->block_mmu_notifications))
2461 return 0;
2462
2463 mutex_lock(&process_info->notifier_lock);
2464 mmu_interval_set_seq(mni, cur_seq);
2465
2466 mem->invalid++;
2467 if (++process_info->evicted_bos == 1) {
2468 /* First eviction, stop the queues */
2469 r = kgd2kfd_quiesce_mm(mni->mm,
2470 KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2471
2472 if (r && r != -ESRCH)
2473 pr_err("Failed to quiesce KFD\n");
2474
2475 if (r != -ESRCH)
2476 queue_delayed_work(system_freezable_wq,
2477 &process_info->restore_userptr_work,
2478 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2479 }
2480 mutex_unlock(&process_info->notifier_lock);
2481
2482 return r;
2483 }
2484
2485 /* Update invalid userptr BOs
2486 *
2487 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2488 * userptr_inval_list and updates user pages for all BOs that have
2489 * been invalidated since their last update.
2490 */
update_invalid_user_pages(struct amdkfd_process_info * process_info,struct mm_struct * mm)2491 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2492 struct mm_struct *mm)
2493 {
2494 struct kgd_mem *mem, *tmp_mem;
2495 struct amdgpu_bo *bo;
2496 struct ttm_operation_ctx ctx = { false, false };
2497 uint32_t invalid;
2498 int ret = 0;
2499
2500 mutex_lock(&process_info->notifier_lock);
2501
2502 /* Move all invalidated BOs to the userptr_inval_list */
2503 list_for_each_entry_safe(mem, tmp_mem,
2504 &process_info->userptr_valid_list,
2505 validate_list)
2506 if (mem->invalid)
2507 list_move_tail(&mem->validate_list,
2508 &process_info->userptr_inval_list);
2509
2510 /* Go through userptr_inval_list and update any invalid user_pages */
2511 list_for_each_entry(mem, &process_info->userptr_inval_list,
2512 validate_list) {
2513 invalid = mem->invalid;
2514 if (!invalid)
2515 /* BO hasn't been invalidated since the last
2516 * revalidation attempt. Keep its page list.
2517 */
2518 continue;
2519
2520 bo = mem->bo;
2521
2522 amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range);
2523 mem->range = NULL;
2524
2525 /* BO reservations and getting user pages (hmm_range_fault)
2526 * must happen outside the notifier lock
2527 */
2528 mutex_unlock(&process_info->notifier_lock);
2529
2530 /* Move the BO to system (CPU) domain if necessary to unmap
2531 * and free the SG table
2532 */
2533 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {
2534 if (amdgpu_bo_reserve(bo, true))
2535 return -EAGAIN;
2536 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
2537 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2538 amdgpu_bo_unreserve(bo);
2539 if (ret) {
2540 pr_err("%s: Failed to invalidate userptr BO\n",
2541 __func__);
2542 return -EAGAIN;
2543 }
2544 }
2545
2546 /* Get updated user pages */
2547 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
2548 &mem->range);
2549 if (ret) {
2550 pr_debug("Failed %d to get user pages\n", ret);
2551
2552 /* Return -EFAULT bad address error as success. It will
2553 * fail later with a VM fault if the GPU tries to access
2554 * it. Better than hanging indefinitely with stalled
2555 * user mode queues.
2556 *
2557 * Return other error -EBUSY or -ENOMEM to retry restore
2558 */
2559 if (ret != -EFAULT)
2560 return ret;
2561
2562 ret = 0;
2563 }
2564
2565 mutex_lock(&process_info->notifier_lock);
2566
2567 /* Mark the BO as valid unless it was invalidated
2568 * again concurrently.
2569 */
2570 if (mem->invalid != invalid) {
2571 ret = -EAGAIN;
2572 goto unlock_out;
2573 }
2574 /* set mem valid if mem has hmm range associated */
2575 if (mem->range)
2576 mem->invalid = 0;
2577 }
2578
2579 unlock_out:
2580 mutex_unlock(&process_info->notifier_lock);
2581
2582 return ret;
2583 }
2584
2585 /* Validate invalid userptr BOs
2586 *
2587 * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables
2588 * with new page addresses and waits for the page table updates to complete.
2589 */
validate_invalid_user_pages(struct amdkfd_process_info * process_info)2590 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2591 {
2592 struct ttm_operation_ctx ctx = { false, false };
2593 struct amdgpu_sync sync;
2594 struct drm_exec exec;
2595
2596 struct amdgpu_vm *peer_vm;
2597 struct kgd_mem *mem, *tmp_mem;
2598 struct amdgpu_bo *bo;
2599 int ret;
2600
2601 amdgpu_sync_create(&sync);
2602
2603 drm_exec_init(&exec, 0, 0);
2604 /* Reserve all BOs and page tables for validation */
2605 drm_exec_until_all_locked(&exec) {
2606 /* Reserve all the page directories */
2607 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2608 vm_list_node) {
2609 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);
2610 drm_exec_retry_on_contention(&exec);
2611 if (unlikely(ret))
2612 goto unreserve_out;
2613 }
2614
2615 /* Reserve the userptr_inval_list entries to resv_list */
2616 list_for_each_entry(mem, &process_info->userptr_inval_list,
2617 validate_list) {
2618 struct drm_gem_object *gobj;
2619
2620 gobj = &mem->bo->tbo.base;
2621 ret = drm_exec_prepare_obj(&exec, gobj, 1);
2622 drm_exec_retry_on_contention(&exec);
2623 if (unlikely(ret))
2624 goto unreserve_out;
2625 }
2626 }
2627
2628 ret = process_validate_vms(process_info, NULL);
2629 if (ret)
2630 goto unreserve_out;
2631
2632 /* Validate BOs and update GPUVM page tables */
2633 list_for_each_entry_safe(mem, tmp_mem,
2634 &process_info->userptr_inval_list,
2635 validate_list) {
2636 struct kfd_mem_attachment *attachment;
2637
2638 bo = mem->bo;
2639
2640 /* Validate the BO if we got user pages */
2641 if (bo->tbo.ttm->pages[0]) {
2642 amdgpu_bo_placement_from_domain(bo, mem->domain);
2643 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2644 if (ret) {
2645 pr_err("%s: failed to validate BO\n", __func__);
2646 goto unreserve_out;
2647 }
2648 }
2649
2650 /* Update mapping. If the BO was not validated
2651 * (because we couldn't get user pages), this will
2652 * clear the page table entries, which will result in
2653 * VM faults if the GPU tries to access the invalid
2654 * memory.
2655 */
2656 list_for_each_entry(attachment, &mem->attachments, list) {
2657 if (!attachment->is_mapped)
2658 continue;
2659
2660 kfd_mem_dmaunmap_attachment(mem, attachment);
2661 ret = update_gpuvm_pte(mem, attachment, &sync);
2662 if (ret) {
2663 pr_err("%s: update PTE failed\n", __func__);
2664 /* make sure this gets validated again */
2665 mutex_lock(&process_info->notifier_lock);
2666 mem->invalid++;
2667 mutex_unlock(&process_info->notifier_lock);
2668 goto unreserve_out;
2669 }
2670 }
2671 }
2672
2673 /* Update page directories */
2674 ret = process_update_pds(process_info, &sync);
2675
2676 unreserve_out:
2677 drm_exec_fini(&exec);
2678 amdgpu_sync_wait(&sync, false);
2679 amdgpu_sync_free(&sync);
2680
2681 return ret;
2682 }
2683
2684 /* Confirm that all user pages are valid while holding the notifier lock
2685 *
2686 * Moves valid BOs from the userptr_inval_list back to userptr_val_list.
2687 */
confirm_valid_user_pages_locked(struct amdkfd_process_info * process_info)2688 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)
2689 {
2690 struct kgd_mem *mem, *tmp_mem;
2691 int ret = 0;
2692
2693 list_for_each_entry_safe(mem, tmp_mem,
2694 &process_info->userptr_inval_list,
2695 validate_list) {
2696 bool valid;
2697
2698 /* keep mem without hmm range at userptr_inval_list */
2699 if (!mem->range)
2700 continue;
2701
2702 /* Only check mem with hmm range associated */
2703 valid = amdgpu_ttm_tt_get_user_pages_done(
2704 mem->bo->tbo.ttm, mem->range);
2705
2706 mem->range = NULL;
2707 if (!valid) {
2708 WARN(!mem->invalid, "Invalid BO not marked invalid");
2709 ret = -EAGAIN;
2710 continue;
2711 }
2712
2713 if (mem->invalid) {
2714 WARN(1, "Valid BO is marked invalid");
2715 ret = -EAGAIN;
2716 continue;
2717 }
2718
2719 list_move_tail(&mem->validate_list,
2720 &process_info->userptr_valid_list);
2721 }
2722
2723 return ret;
2724 }
2725
2726 /* Worker callback to restore evicted userptr BOs
2727 *
2728 * Tries to update and validate all userptr BOs. If successful and no
2729 * concurrent evictions happened, the queues are restarted. Otherwise,
2730 * reschedule for another attempt later.
2731 */
amdgpu_amdkfd_restore_userptr_worker(struct work_struct * work)2732 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2733 {
2734 struct delayed_work *dwork = to_delayed_work(work);
2735 struct amdkfd_process_info *process_info =
2736 container_of(dwork, struct amdkfd_process_info,
2737 restore_userptr_work);
2738 struct task_struct *usertask;
2739 struct mm_struct *mm;
2740 uint32_t evicted_bos;
2741
2742 mutex_lock(&process_info->notifier_lock);
2743 evicted_bos = process_info->evicted_bos;
2744 mutex_unlock(&process_info->notifier_lock);
2745 if (!evicted_bos)
2746 return;
2747
2748 /* Reference task and mm in case of concurrent process termination */
2749 usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
2750 if (!usertask)
2751 return;
2752 mm = get_task_mm(usertask);
2753 if (!mm) {
2754 put_task_struct(usertask);
2755 return;
2756 }
2757
2758 mutex_lock(&process_info->lock);
2759
2760 if (update_invalid_user_pages(process_info, mm))
2761 goto unlock_out;
2762 /* userptr_inval_list can be empty if all evicted userptr BOs
2763 * have been freed. In that case there is nothing to validate
2764 * and we can just restart the queues.
2765 */
2766 if (!list_empty(&process_info->userptr_inval_list)) {
2767 if (validate_invalid_user_pages(process_info))
2768 goto unlock_out;
2769 }
2770 /* Final check for concurrent evicton and atomic update. If
2771 * another eviction happens after successful update, it will
2772 * be a first eviction that calls quiesce_mm. The eviction
2773 * reference counting inside KFD will handle this case.
2774 */
2775 mutex_lock(&process_info->notifier_lock);
2776 if (process_info->evicted_bos != evicted_bos)
2777 goto unlock_notifier_out;
2778
2779 if (confirm_valid_user_pages_locked(process_info)) {
2780 WARN(1, "User pages unexpectedly invalid");
2781 goto unlock_notifier_out;
2782 }
2783
2784 process_info->evicted_bos = evicted_bos = 0;
2785
2786 if (kgd2kfd_resume_mm(mm)) {
2787 pr_err("%s: Failed to resume KFD\n", __func__);
2788 /* No recovery from this failure. Probably the CP is
2789 * hanging. No point trying again.
2790 */
2791 }
2792
2793 unlock_notifier_out:
2794 mutex_unlock(&process_info->notifier_lock);
2795 unlock_out:
2796 mutex_unlock(&process_info->lock);
2797
2798 /* If validation failed, reschedule another attempt */
2799 if (evicted_bos) {
2800 queue_delayed_work(system_freezable_wq,
2801 &process_info->restore_userptr_work,
2802 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2803
2804 kfd_smi_event_queue_restore_rescheduled(mm);
2805 }
2806 mmput(mm);
2807 put_task_struct(usertask);
2808 }
2809
replace_eviction_fence(struct dma_fence __rcu ** ef,struct dma_fence * new_ef)2810 static void replace_eviction_fence(struct dma_fence __rcu **ef,
2811 struct dma_fence *new_ef)
2812 {
2813 struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true
2814 /* protected by process_info->lock */);
2815
2816 /* If we're replacing an unsignaled eviction fence, that fence will
2817 * never be signaled, and if anyone is still waiting on that fence,
2818 * they will hang forever. This should never happen. We should only
2819 * replace the fence in restore_work that only gets scheduled after
2820 * eviction work signaled the fence.
2821 */
2822 WARN_ONCE(!dma_fence_is_signaled(old_ef),
2823 "Replacing unsignaled eviction fence");
2824 dma_fence_put(old_ef);
2825 }
2826
2827 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2828 * KFD process identified by process_info
2829 *
2830 * @process_info: amdkfd_process_info of the KFD process
2831 *
2832 * After memory eviction, restore thread calls this function. The function
2833 * should be called when the Process is still valid. BO restore involves -
2834 *
2835 * 1. Release old eviction fence and create new one
2836 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2837 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2838 * BOs that need to be reserved.
2839 * 4. Reserve all the BOs
2840 * 5. Validate of PD and PT BOs.
2841 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2842 * 7. Add fence to all PD and PT BOs.
2843 * 8. Unreserve all BOs
2844 */
amdgpu_amdkfd_gpuvm_restore_process_bos(void * info,struct dma_fence __rcu ** ef)2845 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef)
2846 {
2847 struct amdkfd_process_info *process_info = info;
2848 struct amdgpu_vm *peer_vm;
2849 struct kgd_mem *mem;
2850 struct list_head duplicate_save;
2851 struct amdgpu_sync sync_obj;
2852 unsigned long failed_size = 0;
2853 unsigned long total_size = 0;
2854 struct drm_exec exec;
2855 int ret;
2856
2857 INIT_LIST_HEAD(&duplicate_save);
2858
2859 mutex_lock(&process_info->lock);
2860
2861 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
2862 drm_exec_until_all_locked(&exec) {
2863 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2864 vm_list_node) {
2865 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);
2866 drm_exec_retry_on_contention(&exec);
2867 if (unlikely(ret)) {
2868 pr_err("Locking VM PD failed, ret: %d\n", ret);
2869 goto ttm_reserve_fail;
2870 }
2871 }
2872
2873 /* Reserve all BOs and page tables/directory. Add all BOs from
2874 * kfd_bo_list to ctx.list
2875 */
2876 list_for_each_entry(mem, &process_info->kfd_bo_list,
2877 validate_list) {
2878 struct drm_gem_object *gobj;
2879
2880 gobj = &mem->bo->tbo.base;
2881 ret = drm_exec_prepare_obj(&exec, gobj, 1);
2882 drm_exec_retry_on_contention(&exec);
2883 if (unlikely(ret)) {
2884 pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret);
2885 goto ttm_reserve_fail;
2886 }
2887 }
2888 }
2889
2890 amdgpu_sync_create(&sync_obj);
2891
2892 /* Validate BOs managed by KFD */
2893 list_for_each_entry(mem, &process_info->kfd_bo_list,
2894 validate_list) {
2895
2896 struct amdgpu_bo *bo = mem->bo;
2897 uint32_t domain = mem->domain;
2898 struct dma_resv_iter cursor;
2899 struct dma_fence *fence;
2900
2901 total_size += amdgpu_bo_size(bo);
2902
2903 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
2904 if (ret) {
2905 pr_debug("Memory eviction: Validate BOs failed\n");
2906 failed_size += amdgpu_bo_size(bo);
2907 ret = amdgpu_amdkfd_bo_validate(bo,
2908 AMDGPU_GEM_DOMAIN_GTT, false);
2909 if (ret) {
2910 pr_debug("Memory eviction: Try again\n");
2911 goto validate_map_fail;
2912 }
2913 }
2914 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2915 DMA_RESV_USAGE_KERNEL, fence) {
2916 ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL);
2917 if (ret) {
2918 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2919 goto validate_map_fail;
2920 }
2921 }
2922 }
2923
2924 if (failed_size)
2925 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2926
2927 /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO
2928 * validations above would invalidate DMABuf imports again.
2929 */
2930 ret = process_validate_vms(process_info, &exec.ticket);
2931 if (ret) {
2932 pr_debug("Validating VMs failed, ret: %d\n", ret);
2933 goto validate_map_fail;
2934 }
2935
2936 /* Update mappings managed by KFD. */
2937 list_for_each_entry(mem, &process_info->kfd_bo_list,
2938 validate_list) {
2939 struct kfd_mem_attachment *attachment;
2940
2941 list_for_each_entry(attachment, &mem->attachments, list) {
2942 if (!attachment->is_mapped)
2943 continue;
2944
2945 kfd_mem_dmaunmap_attachment(mem, attachment);
2946 ret = update_gpuvm_pte(mem, attachment, &sync_obj);
2947 if (ret) {
2948 pr_debug("Memory eviction: update PTE failed. Try again\n");
2949 goto validate_map_fail;
2950 }
2951 }
2952 }
2953
2954 /* Update mappings not managed by KFD */
2955 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2956 vm_list_node) {
2957 struct amdgpu_device *adev = amdgpu_ttm_adev(
2958 peer_vm->root.bo->tbo.bdev);
2959
2960 ret = amdgpu_vm_handle_moved(adev, peer_vm, &exec.ticket);
2961 if (ret) {
2962 pr_debug("Memory eviction: handle moved failed. Try again\n");
2963 goto validate_map_fail;
2964 }
2965 }
2966
2967 /* Update page directories */
2968 ret = process_update_pds(process_info, &sync_obj);
2969 if (ret) {
2970 pr_debug("Memory eviction: update PDs failed. Try again\n");
2971 goto validate_map_fail;
2972 }
2973
2974 /* Sync with fences on all the page tables. They implicitly depend on any
2975 * move fences from amdgpu_vm_handle_moved above.
2976 */
2977 ret = process_sync_pds_resv(process_info, &sync_obj);
2978 if (ret) {
2979 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2980 goto validate_map_fail;
2981 }
2982
2983 /* Wait for validate and PT updates to finish */
2984 amdgpu_sync_wait(&sync_obj, false);
2985
2986 /* The old eviction fence may be unsignaled if restore happens
2987 * after a GPU reset or suspend/resume. Keep the old fence in that
2988 * case. Otherwise release the old eviction fence and create new
2989 * one, because fence only goes from unsignaled to signaled once
2990 * and cannot be reused. Use context and mm from the old fence.
2991 *
2992 * If an old eviction fence signals after this check, that's OK.
2993 * Anyone signaling an eviction fence must stop the queues first
2994 * and schedule another restore worker.
2995 */
2996 if (dma_fence_is_signaled(&process_info->eviction_fence->base)) {
2997 struct amdgpu_amdkfd_fence *new_fence =
2998 amdgpu_amdkfd_fence_create(
2999 process_info->eviction_fence->base.context,
3000 process_info->eviction_fence->mm,
3001 NULL);
3002
3003 if (!new_fence) {
3004 pr_err("Failed to create eviction fence\n");
3005 ret = -ENOMEM;
3006 goto validate_map_fail;
3007 }
3008 dma_fence_put(&process_info->eviction_fence->base);
3009 process_info->eviction_fence = new_fence;
3010 replace_eviction_fence(ef, dma_fence_get(&new_fence->base));
3011 } else {
3012 WARN_ONCE(*ef != &process_info->eviction_fence->base,
3013 "KFD eviction fence doesn't match KGD process_info");
3014 }
3015
3016 /* Attach new eviction fence to all BOs except pinned ones */
3017 list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
3018 if (mem->bo->tbo.pin_count)
3019 continue;
3020
3021 dma_resv_add_fence(mem->bo->tbo.base.resv,
3022 &process_info->eviction_fence->base,
3023 DMA_RESV_USAGE_BOOKKEEP);
3024 }
3025 /* Attach eviction fence to PD / PT BOs and DMABuf imports */
3026 list_for_each_entry(peer_vm, &process_info->vm_list_head,
3027 vm_list_node) {
3028 struct amdgpu_bo *bo = peer_vm->root.bo;
3029
3030 dma_resv_add_fence(bo->tbo.base.resv,
3031 &process_info->eviction_fence->base,
3032 DMA_RESV_USAGE_BOOKKEEP);
3033 }
3034
3035 validate_map_fail:
3036 amdgpu_sync_free(&sync_obj);
3037 ttm_reserve_fail:
3038 drm_exec_fini(&exec);
3039 mutex_unlock(&process_info->lock);
3040 return ret;
3041 }
3042
amdgpu_amdkfd_add_gws_to_process(void * info,void * gws,struct kgd_mem ** mem)3043 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
3044 {
3045 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3046 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
3047 int ret;
3048
3049 if (!info || !gws)
3050 return -EINVAL;
3051
3052 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
3053 if (!*mem)
3054 return -ENOMEM;
3055
3056 mutex_init(&(*mem)->lock);
3057 INIT_LIST_HEAD(&(*mem)->attachments);
3058 (*mem)->bo = amdgpu_bo_ref(gws_bo);
3059 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
3060 (*mem)->process_info = process_info;
3061 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
3062 amdgpu_sync_create(&(*mem)->sync);
3063
3064
3065 /* Validate gws bo the first time it is added to process */
3066 mutex_lock(&(*mem)->process_info->lock);
3067 ret = amdgpu_bo_reserve(gws_bo, false);
3068 if (unlikely(ret)) {
3069 pr_err("Reserve gws bo failed %d\n", ret);
3070 goto bo_reservation_failure;
3071 }
3072
3073 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
3074 if (ret) {
3075 pr_err("GWS BO validate failed %d\n", ret);
3076 goto bo_validation_failure;
3077 }
3078 /* GWS resource is shared b/t amdgpu and amdkfd
3079 * Add process eviction fence to bo so they can
3080 * evict each other.
3081 */
3082 ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);
3083 if (ret)
3084 goto reserve_shared_fail;
3085 dma_resv_add_fence(gws_bo->tbo.base.resv,
3086 &process_info->eviction_fence->base,
3087 DMA_RESV_USAGE_BOOKKEEP);
3088 amdgpu_bo_unreserve(gws_bo);
3089 mutex_unlock(&(*mem)->process_info->lock);
3090
3091 return ret;
3092
3093 reserve_shared_fail:
3094 bo_validation_failure:
3095 amdgpu_bo_unreserve(gws_bo);
3096 bo_reservation_failure:
3097 mutex_unlock(&(*mem)->process_info->lock);
3098 amdgpu_sync_free(&(*mem)->sync);
3099 remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
3100 amdgpu_bo_unref(&gws_bo);
3101 mutex_destroy(&(*mem)->lock);
3102 kfree(*mem);
3103 *mem = NULL;
3104 return ret;
3105 }
3106
amdgpu_amdkfd_remove_gws_from_process(void * info,void * mem)3107 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3108 {
3109 int ret;
3110 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3111 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3112 struct amdgpu_bo *gws_bo = kgd_mem->bo;
3113
3114 /* Remove BO from process's validate list so restore worker won't touch
3115 * it anymore
3116 */
3117 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
3118
3119 ret = amdgpu_bo_reserve(gws_bo, false);
3120 if (unlikely(ret)) {
3121 pr_err("Reserve gws bo failed %d\n", ret);
3122 //TODO add BO back to validate_list?
3123 return ret;
3124 }
3125 amdgpu_amdkfd_remove_eviction_fence(gws_bo,
3126 process_info->eviction_fence);
3127 amdgpu_bo_unreserve(gws_bo);
3128 amdgpu_sync_free(&kgd_mem->sync);
3129 amdgpu_bo_unref(&gws_bo);
3130 mutex_destroy(&kgd_mem->lock);
3131 kfree(mem);
3132 return 0;
3133 }
3134
3135 /* Returns GPU-specific tiling mode information */
amdgpu_amdkfd_get_tile_config(struct amdgpu_device * adev,struct tile_config * config)3136 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
3137 struct tile_config *config)
3138 {
3139 config->gb_addr_config = adev->gfx.config.gb_addr_config;
3140 config->tile_config_ptr = adev->gfx.config.tile_mode_array;
3141 config->num_tile_configs =
3142 ARRAY_SIZE(adev->gfx.config.tile_mode_array);
3143 config->macro_tile_config_ptr =
3144 adev->gfx.config.macrotile_mode_array;
3145 config->num_macro_tile_configs =
3146 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
3147
3148 /* Those values are not set from GFX9 onwards */
3149 config->num_banks = adev->gfx.config.num_banks;
3150 config->num_ranks = adev->gfx.config.num_ranks;
3151
3152 return 0;
3153 }
3154
amdgpu_amdkfd_bo_mapped_to_dev(void * drm_priv,struct kgd_mem * mem)3155 bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem)
3156 {
3157 struct amdgpu_vm *vm = drm_priv_to_vm(drm_priv);
3158 struct kfd_mem_attachment *entry;
3159
3160 list_for_each_entry(entry, &mem->attachments, list) {
3161 if (entry->is_mapped && entry->bo_va->base.vm == vm)
3162 return true;
3163 }
3164 return false;
3165 }
3166
3167 #if defined(CONFIG_DEBUG_FS)
3168
kfd_debugfs_kfd_mem_limits(struct seq_file * m,void * data)3169 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
3170 {
3171
3172 spin_lock(&kfd_mem_limit.mem_limit_lock);
3173 seq_printf(m, "System mem used %lldM out of %lluM\n",
3174 (kfd_mem_limit.system_mem_used >> 20),
3175 (kfd_mem_limit.max_system_mem_limit >> 20));
3176 seq_printf(m, "TTM mem used %lldM out of %lluM\n",
3177 (kfd_mem_limit.ttm_mem_used >> 20),
3178 (kfd_mem_limit.max_ttm_mem_limit >> 20));
3179 spin_unlock(&kfd_mem_limit.mem_limit_lock);
3180
3181 return 0;
3182 }
3183
3184 #endif
3185