1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28 #include <linux/ktime.h>
29 #include <linux/module.h>
30 #include <linux/pagemap.h>
31 #include <linux/pci.h>
32 #include <linux/dma-buf.h>
33
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_exec.h>
37 #include <drm/drm_gem_ttm_helper.h>
38 #include <drm/ttm/ttm_tt.h>
39 #include <drm/drm_syncobj.h>
40
41 #include "amdgpu.h"
42 #include "amdgpu_display.h"
43 #include "amdgpu_dma_buf.h"
44 #include "amdgpu_hmm.h"
45 #include "amdgpu_xgmi.h"
46 #include "amdgpu_vm.h"
47
48 static int
amdgpu_gem_add_input_fence(struct drm_file * filp,uint64_t syncobj_handles_array,uint32_t num_syncobj_handles)49 amdgpu_gem_add_input_fence(struct drm_file *filp,
50 uint64_t syncobj_handles_array,
51 uint32_t num_syncobj_handles)
52 {
53 struct dma_fence *fence;
54 uint32_t *syncobj_handles;
55 int ret, i;
56
57 if (!num_syncobj_handles)
58 return 0;
59
60 syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array),
61 size_mul(sizeof(uint32_t), num_syncobj_handles));
62 if (IS_ERR(syncobj_handles))
63 return PTR_ERR(syncobj_handles);
64
65 for (i = 0; i < num_syncobj_handles; i++) {
66
67 if (!syncobj_handles[i]) {
68 ret = -EINVAL;
69 goto free_memdup;
70 }
71
72 ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence);
73 if (ret)
74 goto free_memdup;
75
76 dma_fence_wait(fence, false);
77
78 /* TODO: optimize async handling */
79 dma_fence_put(fence);
80 }
81
82 free_memdup:
83 kfree(syncobj_handles);
84 return ret;
85 }
86
87 static int
amdgpu_gem_update_timeline_node(struct drm_file * filp,uint32_t syncobj_handle,uint64_t point,struct drm_syncobj ** syncobj,struct dma_fence_chain ** chain)88 amdgpu_gem_update_timeline_node(struct drm_file *filp,
89 uint32_t syncobj_handle,
90 uint64_t point,
91 struct drm_syncobj **syncobj,
92 struct dma_fence_chain **chain)
93 {
94 if (!syncobj_handle)
95 return 0;
96
97 /* Find the sync object */
98 *syncobj = drm_syncobj_find(filp, syncobj_handle);
99 if (!*syncobj)
100 return -ENOENT;
101
102 if (!point)
103 return 0;
104
105 /* Allocate the chain node */
106 *chain = dma_fence_chain_alloc();
107 if (!*chain) {
108 drm_syncobj_put(*syncobj);
109 return -ENOMEM;
110 }
111
112 return 0;
113 }
114
115 static void
amdgpu_gem_update_bo_mapping(struct drm_file * filp,struct amdgpu_bo_va * bo_va,uint32_t operation,uint64_t point,struct dma_fence * fence,struct drm_syncobj * syncobj,struct dma_fence_chain * chain)116 amdgpu_gem_update_bo_mapping(struct drm_file *filp,
117 struct amdgpu_bo_va *bo_va,
118 uint32_t operation,
119 uint64_t point,
120 struct dma_fence *fence,
121 struct drm_syncobj *syncobj,
122 struct dma_fence_chain *chain)
123 {
124 struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL;
125 struct amdgpu_fpriv *fpriv = filp->driver_priv;
126 struct amdgpu_vm *vm = &fpriv->vm;
127 struct dma_fence *last_update;
128
129 if (!syncobj)
130 return;
131
132 /* Find the last update fence */
133 switch (operation) {
134 case AMDGPU_VA_OP_MAP:
135 case AMDGPU_VA_OP_REPLACE:
136 if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv))
137 last_update = vm->last_update;
138 else
139 last_update = bo_va->last_pt_update;
140 break;
141 case AMDGPU_VA_OP_UNMAP:
142 case AMDGPU_VA_OP_CLEAR:
143 last_update = fence;
144 break;
145 default:
146 return;
147 }
148
149 /* Add fence to timeline */
150 if (!point)
151 drm_syncobj_replace_fence(syncobj, last_update);
152 else
153 drm_syncobj_add_point(syncobj, chain, last_update, point);
154 }
155
amdgpu_gem_fault(struct vm_fault * vmf)156 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
157 {
158 struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
159 struct drm_device *ddev = bo->base.dev;
160 vm_fault_t ret;
161 int idx;
162
163 ret = ttm_bo_vm_reserve(bo, vmf);
164 if (ret)
165 return ret;
166
167 if (drm_dev_enter(ddev, &idx)) {
168 ret = amdgpu_bo_fault_reserve_notify(bo);
169 if (ret) {
170 drm_dev_exit(idx);
171 goto unlock;
172 }
173
174 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
175 TTM_BO_VM_NUM_PREFAULT);
176
177 drm_dev_exit(idx);
178 } else {
179 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
180 }
181 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
182 return ret;
183
184 unlock:
185 dma_resv_unlock(bo->base.resv);
186 return ret;
187 }
188
189 static const struct vm_operations_struct amdgpu_gem_vm_ops = {
190 .fault = amdgpu_gem_fault,
191 .open = ttm_bo_vm_open,
192 .close = ttm_bo_vm_close,
193 .access = ttm_bo_vm_access
194 };
195
amdgpu_gem_object_free(struct drm_gem_object * gobj)196 static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
197 {
198 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
199
200 amdgpu_hmm_unregister(aobj);
201 ttm_bo_put(&aobj->tbo);
202 }
203
amdgpu_gem_object_create(struct amdgpu_device * adev,unsigned long size,int alignment,u32 initial_domain,u64 flags,enum ttm_bo_type type,struct dma_resv * resv,struct drm_gem_object ** obj,int8_t xcp_id_plus1)204 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
205 int alignment, u32 initial_domain,
206 u64 flags, enum ttm_bo_type type,
207 struct dma_resv *resv,
208 struct drm_gem_object **obj, int8_t xcp_id_plus1)
209 {
210 struct amdgpu_bo *bo;
211 struct amdgpu_bo_user *ubo;
212 struct amdgpu_bo_param bp;
213 int r;
214
215 memset(&bp, 0, sizeof(bp));
216 *obj = NULL;
217 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
218
219 bp.size = size;
220 bp.byte_align = alignment;
221 bp.type = type;
222 bp.resv = resv;
223 bp.preferred_domain = initial_domain;
224 bp.flags = flags;
225 bp.domain = initial_domain;
226 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
227 bp.xcp_id_plus1 = xcp_id_plus1;
228
229 r = amdgpu_bo_create_user(adev, &bp, &ubo);
230 if (r)
231 return r;
232
233 bo = &ubo->bo;
234 *obj = &bo->tbo.base;
235
236 return 0;
237 }
238
amdgpu_gem_force_release(struct amdgpu_device * adev)239 void amdgpu_gem_force_release(struct amdgpu_device *adev)
240 {
241 struct drm_device *ddev = adev_to_drm(adev);
242 struct drm_file *file;
243
244 mutex_lock(&ddev->filelist_mutex);
245
246 list_for_each_entry(file, &ddev->filelist, lhead) {
247 struct drm_gem_object *gobj;
248 int handle;
249
250 WARN_ONCE(1, "Still active user space clients!\n");
251 spin_lock(&file->table_lock);
252 idr_for_each_entry(&file->object_idr, gobj, handle) {
253 WARN_ONCE(1, "And also active allocations!\n");
254 drm_gem_object_put(gobj);
255 }
256 idr_destroy(&file->object_idr);
257 spin_unlock(&file->table_lock);
258 }
259
260 mutex_unlock(&ddev->filelist_mutex);
261 }
262
263 /*
264 * Call from drm_gem_handle_create which appear in both new and open ioctl
265 * case.
266 */
amdgpu_gem_object_open(struct drm_gem_object * obj,struct drm_file * file_priv)267 static int amdgpu_gem_object_open(struct drm_gem_object *obj,
268 struct drm_file *file_priv)
269 {
270 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
271 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
272 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
273 struct amdgpu_vm *vm = &fpriv->vm;
274 struct amdgpu_bo_va *bo_va;
275 struct mm_struct *mm;
276 int r;
277
278 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
279 if (mm && mm != current->mm)
280 return -EPERM;
281
282 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
283 !amdgpu_vm_is_bo_always_valid(vm, abo))
284 return -EPERM;
285
286 r = amdgpu_bo_reserve(abo, false);
287 if (r)
288 return r;
289
290 amdgpu_vm_bo_update_shared(abo);
291 bo_va = amdgpu_vm_bo_find(vm, abo);
292 if (!bo_va)
293 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
294 else
295 ++bo_va->ref_count;
296
297 /* attach gfx eviction fence */
298 r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
299 if (r) {
300 DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
301 amdgpu_bo_unreserve(abo);
302 return r;
303 }
304
305 amdgpu_bo_unreserve(abo);
306
307 /* Validate and add eviction fence to DMABuf imports with dynamic
308 * attachment in compute VMs. Re-validation will be done by
309 * amdgpu_vm_validate. Fences are on the reservation shared with the
310 * export, which is currently required to be validated and fenced
311 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
312 *
313 * Nested locking below for the case that a GEM object is opened in
314 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
315 * but not for export, this is a different lock class that cannot lead to
316 * circular lock dependencies.
317 */
318 if (!vm->is_compute_context || !vm->process_info)
319 return 0;
320 if (!drm_gem_is_imported(obj) || !dma_buf_is_dynamic(obj->dma_buf))
321 return 0;
322 mutex_lock_nested(&vm->process_info->lock, 1);
323 if (!WARN_ON(!vm->process_info->eviction_fence)) {
324 r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,
325 &vm->process_info->eviction_fence->base);
326 if (r) {
327 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
328
329 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
330 if (ti) {
331 dev_warn(adev->dev, "pid %d\n", ti->task.pid);
332 amdgpu_vm_put_task_info(ti);
333 }
334 }
335 }
336 mutex_unlock(&vm->process_info->lock);
337
338 return r;
339 }
340
amdgpu_gem_object_close(struct drm_gem_object * obj,struct drm_file * file_priv)341 static void amdgpu_gem_object_close(struct drm_gem_object *obj,
342 struct drm_file *file_priv)
343 {
344 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
345 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
346 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
347 struct amdgpu_vm *vm = &fpriv->vm;
348
349 struct dma_fence *fence = NULL;
350 struct amdgpu_bo_va *bo_va;
351 struct drm_exec exec;
352 long r;
353
354 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
355 drm_exec_until_all_locked(&exec) {
356 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
357 drm_exec_retry_on_contention(&exec);
358 if (unlikely(r))
359 goto out_unlock;
360
361 r = amdgpu_vm_lock_pd(vm, &exec, 0);
362 drm_exec_retry_on_contention(&exec);
363 if (unlikely(r))
364 goto out_unlock;
365 }
366
367 if (!amdgpu_vm_is_bo_always_valid(vm, bo))
368 amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
369
370 bo_va = amdgpu_vm_bo_find(vm, bo);
371 if (!bo_va || --bo_va->ref_count)
372 goto out_unlock;
373
374 amdgpu_vm_bo_del(adev, bo_va);
375 amdgpu_vm_bo_update_shared(bo);
376 if (!amdgpu_vm_ready(vm))
377 goto out_unlock;
378
379 r = amdgpu_vm_clear_freed(adev, vm, &fence);
380 if (unlikely(r < 0))
381 dev_err(adev->dev, "failed to clear page "
382 "tables on GEM object close (%ld)\n", r);
383 if (r || !fence)
384 goto out_unlock;
385
386 amdgpu_bo_fence(bo, fence, true);
387 dma_fence_put(fence);
388
389 out_unlock:
390 if (r)
391 dev_err(adev->dev, "leaking bo va (%ld)\n", r);
392 drm_exec_fini(&exec);
393 }
394
amdgpu_gem_object_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)395 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
396 {
397 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
398
399 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
400 return -EPERM;
401 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
402 return -EPERM;
403
404 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
405 * for debugger access to invisible VRAM. Should have used MAP_SHARED
406 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
407 * becoming writable and makes is_cow_mapping(vm_flags) false.
408 */
409 if (is_cow_mapping(vma->vm_flags) &&
410 !(vma->vm_flags & VM_ACCESS_FLAGS))
411 vm_flags_clear(vma, VM_MAYWRITE);
412
413 return drm_gem_ttm_mmap(obj, vma);
414 }
415
416 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
417 .free = amdgpu_gem_object_free,
418 .open = amdgpu_gem_object_open,
419 .close = amdgpu_gem_object_close,
420 .export = amdgpu_gem_prime_export,
421 .vmap = drm_gem_ttm_vmap,
422 .vunmap = drm_gem_ttm_vunmap,
423 .mmap = amdgpu_gem_object_mmap,
424 .vm_ops = &amdgpu_gem_vm_ops,
425 };
426
427 /*
428 * GEM ioctls.
429 */
amdgpu_gem_create_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)430 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
431 struct drm_file *filp)
432 {
433 struct amdgpu_device *adev = drm_to_adev(dev);
434 struct amdgpu_fpriv *fpriv = filp->driver_priv;
435 struct amdgpu_vm *vm = &fpriv->vm;
436 union drm_amdgpu_gem_create *args = data;
437 uint64_t flags = args->in.domain_flags;
438 uint64_t size = args->in.bo_size;
439 struct dma_resv *resv = NULL;
440 struct drm_gem_object *gobj;
441 uint32_t handle, initial_domain;
442 int r;
443
444 /* reject invalid gem flags */
445 if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
446 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
447 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
448 AMDGPU_GEM_CREATE_VRAM_CLEARED |
449 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
450 AMDGPU_GEM_CREATE_EXPLICIT_SYNC |
451 AMDGPU_GEM_CREATE_ENCRYPTED |
452 AMDGPU_GEM_CREATE_GFX12_DCC |
453 AMDGPU_GEM_CREATE_DISCARDABLE))
454 return -EINVAL;
455
456 /* reject invalid gem domains */
457 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
458 return -EINVAL;
459
460 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
461 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
462 return -EINVAL;
463 }
464
465 /* always clear VRAM */
466 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
467
468 /* create a gem object to contain this object in */
469 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
470 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
471 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
472 /* if gds bo is created from user space, it must be
473 * passed to bo list
474 */
475 DRM_ERROR("GDS bo cannot be per-vm-bo\n");
476 return -EINVAL;
477 }
478 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
479 }
480
481 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
482 r = amdgpu_bo_reserve(vm->root.bo, false);
483 if (r)
484 return r;
485
486 resv = vm->root.bo->tbo.base.resv;
487 }
488
489 initial_domain = (u32)(0xffffffff & args->in.domains);
490 retry:
491 r = amdgpu_gem_object_create(adev, size, args->in.alignment,
492 initial_domain,
493 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
494 if (r && r != -ERESTARTSYS) {
495 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
496 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
497 goto retry;
498 }
499
500 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
501 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
502 goto retry;
503 }
504 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
505 size, initial_domain, args->in.alignment, r);
506 }
507
508 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
509 if (!r) {
510 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
511
512 abo->parent = amdgpu_bo_ref(vm->root.bo);
513 }
514 amdgpu_bo_unreserve(vm->root.bo);
515 }
516 if (r)
517 return r;
518
519 r = drm_gem_handle_create(filp, gobj, &handle);
520 /* drop reference from allocate - handle holds it now */
521 drm_gem_object_put(gobj);
522 if (r)
523 return r;
524
525 memset(args, 0, sizeof(*args));
526 args->out.handle = handle;
527 return 0;
528 }
529
amdgpu_gem_userptr_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)530 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
531 struct drm_file *filp)
532 {
533 struct ttm_operation_ctx ctx = { true, false };
534 struct amdgpu_device *adev = drm_to_adev(dev);
535 struct drm_amdgpu_gem_userptr *args = data;
536 struct amdgpu_fpriv *fpriv = filp->driver_priv;
537 struct drm_gem_object *gobj;
538 struct hmm_range *range;
539 struct amdgpu_bo *bo;
540 uint32_t handle;
541 int r;
542
543 args->addr = untagged_addr(args->addr);
544
545 if (offset_in_page(args->addr | args->size))
546 return -EINVAL;
547
548 /* reject unknown flag values */
549 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
550 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
551 AMDGPU_GEM_USERPTR_REGISTER))
552 return -EINVAL;
553
554 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
555 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
556
557 /* if we want to write to it we must install a MMU notifier */
558 return -EACCES;
559 }
560
561 /* create a gem object to contain this object in */
562 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
563 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
564 if (r)
565 return r;
566
567 bo = gem_to_amdgpu_bo(gobj);
568 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
569 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
570 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
571 if (r)
572 goto release_object;
573
574 r = amdgpu_hmm_register(bo, args->addr);
575 if (r)
576 goto release_object;
577
578 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
579 r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
580 &range);
581 if (r)
582 goto release_object;
583
584 r = amdgpu_bo_reserve(bo, true);
585 if (r)
586 goto user_pages_done;
587
588 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
589 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
590 amdgpu_bo_unreserve(bo);
591 if (r)
592 goto user_pages_done;
593 }
594
595 r = drm_gem_handle_create(filp, gobj, &handle);
596 if (r)
597 goto user_pages_done;
598
599 args->handle = handle;
600
601 user_pages_done:
602 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
603 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
604
605 release_object:
606 drm_gem_object_put(gobj);
607
608 return r;
609 }
610
amdgpu_mode_dumb_mmap(struct drm_file * filp,struct drm_device * dev,uint32_t handle,uint64_t * offset_p)611 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
612 struct drm_device *dev,
613 uint32_t handle, uint64_t *offset_p)
614 {
615 struct drm_gem_object *gobj;
616 struct amdgpu_bo *robj;
617
618 gobj = drm_gem_object_lookup(filp, handle);
619 if (!gobj)
620 return -ENOENT;
621
622 robj = gem_to_amdgpu_bo(gobj);
623 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
624 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
625 drm_gem_object_put(gobj);
626 return -EPERM;
627 }
628 *offset_p = amdgpu_bo_mmap_offset(robj);
629 drm_gem_object_put(gobj);
630 return 0;
631 }
632
amdgpu_gem_mmap_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)633 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
634 struct drm_file *filp)
635 {
636 union drm_amdgpu_gem_mmap *args = data;
637 uint32_t handle = args->in.handle;
638
639 memset(args, 0, sizeof(*args));
640 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
641 }
642
643 /**
644 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
645 *
646 * @timeout_ns: timeout in ns
647 *
648 * Calculate the timeout in jiffies from an absolute timeout in ns.
649 */
amdgpu_gem_timeout(uint64_t timeout_ns)650 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
651 {
652 unsigned long timeout_jiffies;
653 ktime_t timeout;
654
655 /* clamp timeout if it's to large */
656 if (((int64_t)timeout_ns) < 0)
657 return MAX_SCHEDULE_TIMEOUT;
658
659 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
660 if (ktime_to_ns(timeout) < 0)
661 return 0;
662
663 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
664 /* clamp timeout to avoid unsigned-> signed overflow */
665 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)
666 return MAX_SCHEDULE_TIMEOUT - 1;
667
668 return timeout_jiffies;
669 }
670
amdgpu_gem_wait_idle_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)671 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
672 struct drm_file *filp)
673 {
674 union drm_amdgpu_gem_wait_idle *args = data;
675 struct drm_gem_object *gobj;
676 struct amdgpu_bo *robj;
677 uint32_t handle = args->in.handle;
678 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
679 int r = 0;
680 long ret;
681
682 gobj = drm_gem_object_lookup(filp, handle);
683 if (!gobj)
684 return -ENOENT;
685
686 robj = gem_to_amdgpu_bo(gobj);
687 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
688 true, timeout);
689
690 /* ret == 0 means not signaled,
691 * ret > 0 means signaled
692 * ret < 0 means interrupted before timeout
693 */
694 if (ret >= 0) {
695 memset(args, 0, sizeof(*args));
696 args->out.status = (ret == 0);
697 } else
698 r = ret;
699
700 drm_gem_object_put(gobj);
701 return r;
702 }
703
amdgpu_gem_metadata_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)704 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
705 struct drm_file *filp)
706 {
707 struct drm_amdgpu_gem_metadata *args = data;
708 struct drm_gem_object *gobj;
709 struct amdgpu_bo *robj;
710 int r = -1;
711
712 DRM_DEBUG("%d\n", args->handle);
713 gobj = drm_gem_object_lookup(filp, args->handle);
714 if (gobj == NULL)
715 return -ENOENT;
716 robj = gem_to_amdgpu_bo(gobj);
717
718 r = amdgpu_bo_reserve(robj, false);
719 if (unlikely(r != 0))
720 goto out;
721
722 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
723 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
724 r = amdgpu_bo_get_metadata(robj, args->data.data,
725 sizeof(args->data.data),
726 &args->data.data_size_bytes,
727 &args->data.flags);
728 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
729 if (args->data.data_size_bytes > sizeof(args->data.data)) {
730 r = -EINVAL;
731 goto unreserve;
732 }
733 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
734 if (!r)
735 r = amdgpu_bo_set_metadata(robj, args->data.data,
736 args->data.data_size_bytes,
737 args->data.flags);
738 }
739
740 unreserve:
741 amdgpu_bo_unreserve(robj);
742 out:
743 drm_gem_object_put(gobj);
744 return r;
745 }
746
747 /**
748 * amdgpu_gem_va_update_vm -update the bo_va in its VM
749 *
750 * @adev: amdgpu_device pointer
751 * @vm: vm to update
752 * @bo_va: bo_va to update
753 * @operation: map, unmap or clear
754 *
755 * Update the bo_va directly after setting its address. Errors are not
756 * vital here, so they are not reported back to userspace.
757 *
758 * Returns resulting fence if freed BO(s) got cleared from the PT.
759 * otherwise stub fence in case of error.
760 */
761 static struct dma_fence *
amdgpu_gem_va_update_vm(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_va * bo_va,uint32_t operation)762 amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
763 struct amdgpu_vm *vm,
764 struct amdgpu_bo_va *bo_va,
765 uint32_t operation)
766 {
767 struct dma_fence *fence = dma_fence_get_stub();
768 int r;
769
770 if (!amdgpu_vm_ready(vm))
771 return fence;
772
773 r = amdgpu_vm_clear_freed(adev, vm, &fence);
774 if (r)
775 goto error;
776
777 if (operation == AMDGPU_VA_OP_MAP ||
778 operation == AMDGPU_VA_OP_REPLACE) {
779 r = amdgpu_vm_bo_update(adev, bo_va, false);
780 if (r)
781 goto error;
782 }
783
784 r = amdgpu_vm_update_pdes(adev, vm, false);
785
786 error:
787 if (r && r != -ERESTARTSYS)
788 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
789
790 return fence;
791 }
792
793 /**
794 * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
795 *
796 * @adev: amdgpu_device pointer
797 * @flags: GEM UAPI flags
798 *
799 * Returns the GEM UAPI flags mapped into hardware for the ASIC.
800 */
amdgpu_gem_va_map_flags(struct amdgpu_device * adev,uint32_t flags)801 uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
802 {
803 uint64_t pte_flag = 0;
804
805 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
806 pte_flag |= AMDGPU_PTE_EXECUTABLE;
807 if (flags & AMDGPU_VM_PAGE_READABLE)
808 pte_flag |= AMDGPU_PTE_READABLE;
809 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
810 pte_flag |= AMDGPU_PTE_WRITEABLE;
811 if (flags & AMDGPU_VM_PAGE_PRT)
812 pte_flag |= AMDGPU_PTE_PRT_FLAG(adev);
813 if (flags & AMDGPU_VM_PAGE_NOALLOC)
814 pte_flag |= AMDGPU_PTE_NOALLOC;
815
816 if (adev->gmc.gmc_funcs->map_mtype)
817 pte_flag |= amdgpu_gmc_map_mtype(adev,
818 flags & AMDGPU_VM_MTYPE_MASK);
819
820 return pte_flag;
821 }
822
amdgpu_gem_va_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)823 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
824 struct drm_file *filp)
825 {
826 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
827 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
828 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
829 AMDGPU_VM_PAGE_NOALLOC;
830 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
831 AMDGPU_VM_PAGE_PRT;
832
833 struct drm_amdgpu_gem_va *args = data;
834 struct drm_gem_object *gobj;
835 struct amdgpu_device *adev = drm_to_adev(dev);
836 struct amdgpu_fpriv *fpriv = filp->driver_priv;
837 struct amdgpu_bo *abo;
838 struct amdgpu_bo_va *bo_va;
839 struct drm_syncobj *timeline_syncobj = NULL;
840 struct dma_fence_chain *timeline_chain = NULL;
841 struct dma_fence *fence;
842 struct drm_exec exec;
843 uint64_t va_flags;
844 uint64_t vm_size;
845 int r = 0;
846
847 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
848 dev_dbg(dev->dev,
849 "va_address 0x%llx is in reserved area 0x%llx\n",
850 args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
851 return -EINVAL;
852 }
853
854 if (args->va_address >= AMDGPU_GMC_HOLE_START &&
855 args->va_address < AMDGPU_GMC_HOLE_END) {
856 dev_dbg(dev->dev,
857 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
858 args->va_address, AMDGPU_GMC_HOLE_START,
859 AMDGPU_GMC_HOLE_END);
860 return -EINVAL;
861 }
862
863 args->va_address &= AMDGPU_GMC_HOLE_MASK;
864
865 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
866 vm_size -= AMDGPU_VA_RESERVED_TOP;
867 if (args->va_address + args->map_size > vm_size) {
868 dev_dbg(dev->dev,
869 "va_address 0x%llx is in top reserved area 0x%llx\n",
870 args->va_address + args->map_size, vm_size);
871 return -EINVAL;
872 }
873
874 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
875 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
876 args->flags);
877 return -EINVAL;
878 }
879
880 switch (args->operation) {
881 case AMDGPU_VA_OP_MAP:
882 case AMDGPU_VA_OP_UNMAP:
883 case AMDGPU_VA_OP_CLEAR:
884 case AMDGPU_VA_OP_REPLACE:
885 break;
886 default:
887 dev_dbg(dev->dev, "unsupported operation %d\n",
888 args->operation);
889 return -EINVAL;
890 }
891
892 if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
893 !(args->flags & AMDGPU_VM_PAGE_PRT)) {
894 gobj = drm_gem_object_lookup(filp, args->handle);
895 if (gobj == NULL)
896 return -ENOENT;
897 abo = gem_to_amdgpu_bo(gobj);
898 } else {
899 gobj = NULL;
900 abo = NULL;
901 }
902
903 r = amdgpu_gem_add_input_fence(filp,
904 args->input_fence_syncobj_handles,
905 args->num_syncobj_handles);
906 if (r)
907 goto error_put_gobj;
908
909 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
910 DRM_EXEC_IGNORE_DUPLICATES, 0);
911 drm_exec_until_all_locked(&exec) {
912 if (gobj) {
913 r = drm_exec_lock_obj(&exec, gobj);
914 drm_exec_retry_on_contention(&exec);
915 if (unlikely(r))
916 goto error;
917 }
918
919 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
920 drm_exec_retry_on_contention(&exec);
921 if (unlikely(r))
922 goto error;
923 }
924
925 if (abo) {
926 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
927 if (!bo_va) {
928 r = -ENOENT;
929 goto error;
930 }
931 } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
932 bo_va = fpriv->prt_va;
933 } else {
934 bo_va = NULL;
935 }
936
937 r = amdgpu_gem_update_timeline_node(filp,
938 args->vm_timeline_syncobj_out,
939 args->vm_timeline_point,
940 &timeline_syncobj,
941 &timeline_chain);
942 if (r)
943 goto error;
944
945 switch (args->operation) {
946 case AMDGPU_VA_OP_MAP:
947 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
948 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
949 args->offset_in_bo, args->map_size,
950 va_flags);
951 break;
952 case AMDGPU_VA_OP_UNMAP:
953 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
954 break;
955
956 case AMDGPU_VA_OP_CLEAR:
957 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
958 args->va_address,
959 args->map_size);
960 break;
961 case AMDGPU_VA_OP_REPLACE:
962 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
963 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
964 args->offset_in_bo, args->map_size,
965 va_flags);
966 break;
967 default:
968 break;
969 }
970 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) {
971 fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
972 args->operation);
973
974 if (timeline_syncobj)
975 amdgpu_gem_update_bo_mapping(filp, bo_va,
976 args->operation,
977 args->vm_timeline_point,
978 fence, timeline_syncobj,
979 timeline_chain);
980 else
981 dma_fence_put(fence);
982
983 }
984
985 error:
986 drm_exec_fini(&exec);
987 error_put_gobj:
988 drm_gem_object_put(gobj);
989 return r;
990 }
991
amdgpu_gem_op_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)992 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
993 struct drm_file *filp)
994 {
995 struct drm_amdgpu_gem_op *args = data;
996 struct drm_gem_object *gobj;
997 struct amdgpu_vm_bo_base *base;
998 struct amdgpu_bo *robj;
999 int r;
1000
1001 gobj = drm_gem_object_lookup(filp, args->handle);
1002 if (!gobj)
1003 return -ENOENT;
1004
1005 robj = gem_to_amdgpu_bo(gobj);
1006
1007 r = amdgpu_bo_reserve(robj, false);
1008 if (unlikely(r))
1009 goto out;
1010
1011 switch (args->op) {
1012 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
1013 struct drm_amdgpu_gem_create_in info;
1014 void __user *out = u64_to_user_ptr(args->value);
1015
1016 info.bo_size = robj->tbo.base.size;
1017 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
1018 info.domains = robj->preferred_domains;
1019 info.domain_flags = robj->flags;
1020 amdgpu_bo_unreserve(robj);
1021 if (copy_to_user(out, &info, sizeof(info)))
1022 r = -EFAULT;
1023 break;
1024 }
1025 case AMDGPU_GEM_OP_SET_PLACEMENT:
1026 if (drm_gem_is_imported(&robj->tbo.base) &&
1027 args->value & AMDGPU_GEM_DOMAIN_VRAM) {
1028 r = -EINVAL;
1029 amdgpu_bo_unreserve(robj);
1030 break;
1031 }
1032 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1033 r = -EPERM;
1034 amdgpu_bo_unreserve(robj);
1035 break;
1036 }
1037 for (base = robj->vm_bo; base; base = base->next)
1038 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1039 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1040 r = -EINVAL;
1041 amdgpu_bo_unreserve(robj);
1042 goto out;
1043 }
1044
1045
1046 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1047 AMDGPU_GEM_DOMAIN_GTT |
1048 AMDGPU_GEM_DOMAIN_CPU);
1049 robj->allowed_domains = robj->preferred_domains;
1050 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1051 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1052
1053 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1054 amdgpu_vm_bo_invalidate(robj, true);
1055
1056 amdgpu_bo_unreserve(robj);
1057 break;
1058 default:
1059 amdgpu_bo_unreserve(robj);
1060 r = -EINVAL;
1061 }
1062
1063 out:
1064 drm_gem_object_put(gobj);
1065 return r;
1066 }
1067
amdgpu_gem_align_pitch(struct amdgpu_device * adev,int width,int cpp,bool tiled)1068 static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1069 int width,
1070 int cpp,
1071 bool tiled)
1072 {
1073 int aligned = width;
1074 int pitch_mask = 0;
1075
1076 switch (cpp) {
1077 case 1:
1078 pitch_mask = 255;
1079 break;
1080 case 2:
1081 pitch_mask = 127;
1082 break;
1083 case 3:
1084 case 4:
1085 pitch_mask = 63;
1086 break;
1087 }
1088
1089 aligned += pitch_mask;
1090 aligned &= ~pitch_mask;
1091 return aligned * cpp;
1092 }
1093
amdgpu_mode_dumb_create(struct drm_file * file_priv,struct drm_device * dev,struct drm_mode_create_dumb * args)1094 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1095 struct drm_device *dev,
1096 struct drm_mode_create_dumb *args)
1097 {
1098 struct amdgpu_device *adev = drm_to_adev(dev);
1099 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1100 struct drm_gem_object *gobj;
1101 uint32_t handle;
1102 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1103 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1104 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1105 u32 domain;
1106 int r;
1107
1108 /*
1109 * The buffer returned from this function should be cleared, but
1110 * it can only be done if the ring is enabled or we'll fail to
1111 * create the buffer.
1112 */
1113 if (adev->mman.buffer_funcs_enabled)
1114 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1115
1116 args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1117 DIV_ROUND_UP(args->bpp, 8), 0);
1118 args->size = (u64)args->pitch * args->height;
1119 args->size = ALIGN(args->size, PAGE_SIZE);
1120 domain = amdgpu_bo_get_preferred_domain(adev,
1121 amdgpu_display_supported_domains(adev, flags));
1122 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1123 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1124 if (r)
1125 return -ENOMEM;
1126
1127 r = drm_gem_handle_create(file_priv, gobj, &handle);
1128 /* drop reference from allocate - handle holds it now */
1129 drm_gem_object_put(gobj);
1130 if (r)
1131 return r;
1132
1133 args->handle = handle;
1134 return 0;
1135 }
1136
1137 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_gem_info_show(struct seq_file * m,void * unused)1138 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1139 {
1140 struct amdgpu_device *adev = m->private;
1141 struct drm_device *dev = adev_to_drm(adev);
1142 struct drm_file *file;
1143 int r;
1144
1145 r = mutex_lock_interruptible(&dev->filelist_mutex);
1146 if (r)
1147 return r;
1148
1149 list_for_each_entry(file, &dev->filelist, lhead) {
1150 struct task_struct *task;
1151 struct drm_gem_object *gobj;
1152 struct pid *pid;
1153 int id;
1154
1155 /*
1156 * Although we have a valid reference on file->pid, that does
1157 * not guarantee that the task_struct who called get_pid() is
1158 * still alive (e.g. get_pid(current) => fork() => exit()).
1159 * Therefore, we need to protect this ->comm access using RCU.
1160 */
1161 rcu_read_lock();
1162 pid = rcu_dereference(file->pid);
1163 task = pid_task(pid, PIDTYPE_TGID);
1164 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1165 task ? task->comm : "<unknown>");
1166 rcu_read_unlock();
1167
1168 spin_lock(&file->table_lock);
1169 idr_for_each_entry(&file->object_idr, gobj, id) {
1170 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1171
1172 amdgpu_bo_print_info(id, bo, m);
1173 }
1174 spin_unlock(&file->table_lock);
1175 }
1176
1177 mutex_unlock(&dev->filelist_mutex);
1178 return 0;
1179 }
1180
1181 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1182
1183 #endif
1184
amdgpu_debugfs_gem_init(struct amdgpu_device * adev)1185 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1186 {
1187 #if defined(CONFIG_DEBUG_FS)
1188 struct drm_minor *minor = adev_to_drm(adev)->primary;
1189 struct dentry *root = minor->debugfs_root;
1190
1191 debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1192 &amdgpu_debugfs_gem_info_fops);
1193 #endif
1194 }
1195