829 __restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,  in __restore_vm_bo_list()  argument
871 __drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,  in __drm_gpuvm_bo_list_add()  argument
921 __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,  in __drm_gpuvm_bo_list_del()  argument
976 static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
989 drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)  in drm_gpuvm_warn_check_overflow()  argument
991 	return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),  in drm_gpuvm_warn_check_overflow()
996 drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range)  in drm_gpuvm_in_mm_range()  argument
999 	u64 mm_start = gpuvm->mm_start;  in drm_gpuvm_in_mm_range()
1000 	u64 mm_end = mm_start + gpuvm->mm_range;  in drm_gpuvm_in_mm_range()
1006 drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)  in drm_gpuvm_in_kernel_node()  argument
1009 	u64 kstart = gpuvm->kernel_alloc_node.va.addr;  in drm_gpuvm_in_kernel_node()
1010 	u64 krange = gpuvm->kernel_alloc_node.va.range;  in drm_gpuvm_in_kernel_node()
1028 drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,  in drm_gpuvm_range_valid()  argument
1032 	       drm_gpuvm_in_mm_range(gpuvm, addr, range) &&  in drm_gpuvm_range_valid()
1033 	       !drm_gpuvm_in_kernel_node(gpuvm, addr, range);  in drm_gpuvm_range_valid()
1093 drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,  in drm_gpuvm_init()  argument
1101 	gpuvm->rb.tree = RB_ROOT_CACHED;  in drm_gpuvm_init()
1102 	INIT_LIST_HEAD(&gpuvm->rb.list);  in drm_gpuvm_init()
1104 	INIT_LIST_HEAD(&gpuvm->extobj.list);  in drm_gpuvm_init()
1105 	spin_lock_init(&gpuvm->extobj.lock);  in drm_gpuvm_init()
1107 	INIT_LIST_HEAD(&gpuvm->evict.list);  in drm_gpuvm_init()
1108 	spin_lock_init(&gpuvm->evict.lock);  in drm_gpuvm_init()
1110 	init_llist_head(&gpuvm->bo_defer);  in drm_gpuvm_init()
1112 	kref_init(&gpuvm->kref);  in drm_gpuvm_init()
1114 	gpuvm->name = name ? name : "unknown";  in drm_gpuvm_init()
1115 	gpuvm->flags = flags;  in drm_gpuvm_init()
1116 	gpuvm->ops = ops;  in drm_gpuvm_init()
1117 	gpuvm->drm = drm;  in drm_gpuvm_init()
1118 	gpuvm->r_obj = r_obj;  in drm_gpuvm_init()
1122 	drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);  in drm_gpuvm_init()
1123 	gpuvm->mm_start = start_offset;  in drm_gpuvm_init()
1124 	gpuvm->mm_range = range;  in drm_gpuvm_init()
1126 	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));  in drm_gpuvm_init()
1128 		gpuvm->kernel_alloc_node.va.addr = reserve_offset;  in drm_gpuvm_init()
1129 		gpuvm->kernel_alloc_node.va.range = reserve_range;  in drm_gpuvm_init()
1131 		if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,  in drm_gpuvm_init()
1133 			__drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node);  in drm_gpuvm_init()
1139 drm_gpuvm_fini(struct drm_gpuvm *gpuvm)  in drm_gpuvm_fini()  argument
1141 	gpuvm->name = NULL;  in drm_gpuvm_fini()
1143 	if (gpuvm->kernel_alloc_node.va.range)  in drm_gpuvm_fini()
1144 		__drm_gpuva_remove(&gpuvm->kernel_alloc_node);  in drm_gpuvm_fini()
1146 	drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),  in drm_gpuvm_fini()
1149 	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),  in drm_gpuvm_fini()
1151 	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),  in drm_gpuvm_fini()
1153 	drm_WARN(gpuvm->drm, !llist_empty(&gpuvm->bo_defer),  in drm_gpuvm_fini()
1156 	drm_gem_object_put(gpuvm->r_obj);  in drm_gpuvm_fini()
1162 	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);  in drm_gpuvm_free()  local
1164 	drm_gpuvm_fini(gpuvm);  in drm_gpuvm_free()
1166 	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))  in drm_gpuvm_free()
1169 	gpuvm->ops->vm_free(gpuvm);  in drm_gpuvm_free()
1181 drm_gpuvm_put(struct drm_gpuvm *gpuvm)  in drm_gpuvm_put()  argument
1183 	if (gpuvm)  in drm_gpuvm_put()
1184 		kref_put(&gpuvm->kref, drm_gpuvm_free);  in drm_gpuvm_put()
1211 drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,  in drm_gpuvm_prepare_vm()  argument
1215 	return exec_prepare_obj(exec, gpuvm->r_obj, num_fences);  in drm_gpuvm_prepare_vm()
1220 __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,  in __drm_gpuvm_prepare_objects()  argument
1228 	for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {  in __drm_gpuvm_prepare_objects()
1235 	restore_vm_bo_list(gpuvm, extobj);  in __drm_gpuvm_prepare_objects()
1241 drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,  in drm_gpuvm_prepare_objects_locked()  argument
1248 	drm_gpuvm_resv_assert_held(gpuvm);  in drm_gpuvm_prepare_objects_locked()
1249 	list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {  in drm_gpuvm_prepare_objects_locked()
1288 drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,  in drm_gpuvm_prepare_objects()  argument
1292 	if (drm_gpuvm_resv_protected(gpuvm))  in drm_gpuvm_prepare_objects()
1293 		return drm_gpuvm_prepare_objects_locked(gpuvm, exec,  in drm_gpuvm_prepare_objects()
1296 		return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);  in drm_gpuvm_prepare_objects()
1315 drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,  in drm_gpuvm_prepare_range()  argument
1322 	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {  in drm_gpuvm_prepare_range()
1351 	struct drm_gpuvm *gpuvm = vm_exec->vm;  in drm_gpuvm_exec_lock()  local
1359 		ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);  in drm_gpuvm_exec_lock()
1364 		ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);  in drm_gpuvm_exec_lock()
1443 	struct drm_gpuvm *gpuvm = vm_exec->vm;  in drm_gpuvm_exec_lock_range()  local
1450 		ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,  in drm_gpuvm_exec_lock_range()
1466 __drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)  in __drm_gpuvm_validate()  argument
1468 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in __drm_gpuvm_validate()
1473 	for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {  in __drm_gpuvm_validate()
1480 	restore_vm_bo_list(gpuvm, evict);  in __drm_gpuvm_validate()
1486 drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)  in drm_gpuvm_validate_locked()  argument
1488 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_validate_locked()
1492 	drm_gpuvm_resv_assert_held(gpuvm);  in drm_gpuvm_validate_locked()
1494 	list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,  in drm_gpuvm_validate_locked()
1522 drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)  in drm_gpuvm_validate()  argument
1524 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_validate()
1529 	if (drm_gpuvm_resv_protected(gpuvm))  in drm_gpuvm_validate()
1530 		return drm_gpuvm_validate_locked(gpuvm, exec);  in drm_gpuvm_validate()
1532 		return __drm_gpuvm_validate(gpuvm, exec);  in drm_gpuvm_validate()
1546 drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,  in drm_gpuvm_resv_add_fence()  argument
1558 				   drm_gpuvm_is_extobj(gpuvm, obj) ?  in drm_gpuvm_resv_add_fence()
1575 drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,  in drm_gpuvm_bo_create()  argument
1578 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_bo_create()
1589 	vm_bo->vm = drm_gpuvm_get(gpuvm);  in drm_gpuvm_bo_create()
1619 	struct drm_gpuvm *gpuvm = vm_bo->vm;  in drm_gpuvm_bo_destroy_not_in_lists()  local
1620 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_bo_destroy_not_in_lists()
1628 	drm_gpuvm_put(gpuvm);  in drm_gpuvm_bo_destroy_not_in_lists()
1646 	struct drm_gpuvm *gpuvm = vm_bo->vm;  in drm_gpuvm_bo_destroy()  local
1647 	bool lock = !drm_gpuvm_resv_protected(gpuvm);  in drm_gpuvm_bo_destroy()
1650 		drm_gpuvm_resv_assert_held(gpuvm);  in drm_gpuvm_bo_destroy()
1655 	drm_gem_gpuva_assert_lock_held(gpuvm, vm_bo->obj);  in drm_gpuvm_bo_destroy()
1774 drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm)  in drm_gpuvm_bo_deferred_cleanup()  argument
1779 	bo_defer = llist_del_all(&gpuvm->bo_defer);  in drm_gpuvm_bo_deferred_cleanup()
1783 	if (drm_gpuvm_resv_protected(gpuvm)) {  in drm_gpuvm_bo_deferred_cleanup()
1784 		dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL);  in drm_gpuvm_bo_deferred_cleanup()
1789 		dma_resv_unlock(drm_gpuvm_resv(gpuvm));  in drm_gpuvm_bo_deferred_cleanup()
1801 __drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,  in __drm_gpuvm_bo_find()  argument
1806 	drm_gem_gpuva_assert_lock_held(gpuvm, obj);  in __drm_gpuvm_bo_find()
1808 		if (vm_bo->vm == gpuvm)  in __drm_gpuvm_bo_find()
1827 drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,  in drm_gpuvm_bo_find()  argument
1830 	struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);  in drm_gpuvm_bo_find()
1854 drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm,  in drm_gpuvm_bo_obtain_locked()  argument
1865 	drm_WARN_ON(gpuvm->drm, drm_gpuvm_immediate_mode(gpuvm));  in drm_gpuvm_bo_obtain_locked()
1867 	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);  in drm_gpuvm_bo_obtain_locked()
1871 	vm_bo = drm_gpuvm_bo_create(gpuvm, obj);  in drm_gpuvm_bo_obtain_locked()
1875 	drm_gem_gpuva_assert_lock_held(gpuvm, obj);  in drm_gpuvm_bo_obtain_locked()
1904 	struct drm_gpuvm *gpuvm = __vm_bo->vm;  in drm_gpuvm_bo_obtain_prealloc()  local
1908 	drm_WARN_ON(gpuvm->drm, !drm_gpuvm_immediate_mode(gpuvm));  in drm_gpuvm_bo_obtain_prealloc()
1911 	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);  in drm_gpuvm_bo_obtain_prealloc()
1918 	drm_gem_gpuva_assert_lock_held(gpuvm, obj);  in drm_gpuvm_bo_obtain_prealloc()
1938 	struct drm_gpuvm *gpuvm = vm_bo->vm;  in drm_gpuvm_bo_extobj_add()  local
1939 	bool lock = !drm_gpuvm_resv_protected(gpuvm);  in drm_gpuvm_bo_extobj_add()
1942 		drm_gpuvm_resv_assert_held(gpuvm);  in drm_gpuvm_bo_extobj_add()
1944 	if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj))  in drm_gpuvm_bo_extobj_add()
1960 	struct drm_gpuvm *gpuvm = vm_bo->vm;  in drm_gpuvm_bo_evict()  local
1962 	bool lock = !drm_gpuvm_resv_protected(gpuvm);  in drm_gpuvm_bo_evict()
1971 	if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)  in drm_gpuvm_bo_evict()
1982 __drm_gpuva_insert(struct drm_gpuvm *gpuvm,  in __drm_gpuva_insert()  argument
1988 	if (drm_gpuva_it_iter_first(&gpuvm->rb.tree,  in __drm_gpuva_insert()
1993 	va->vm = gpuvm;  in __drm_gpuva_insert()
1995 	drm_gpuva_it_insert(va, &gpuvm->rb.tree);  in __drm_gpuva_insert()
2001 		head = &gpuvm->rb.list;  in __drm_gpuva_insert()
2023 drm_gpuva_insert(struct drm_gpuvm *gpuvm,  in drm_gpuva_insert()  argument
2030 	if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))  in drm_gpuva_insert()
2033 	ret = __drm_gpuva_insert(gpuvm, va);  in drm_gpuva_insert()
2040 		drm_gpuvm_get(gpuvm);  in drm_gpuva_insert()
2066 	struct drm_gpuvm *gpuvm = va->vm;  in drm_gpuva_remove()  local
2068 	if (unlikely(va == &gpuvm->kernel_alloc_node)) {  in drm_gpuva_remove()
2069 		drm_WARN(gpuvm->drm, 1,  in drm_gpuva_remove()
2097 	struct drm_gpuvm *gpuvm = va->vm;  in drm_gpuva_link()  local
2102 	drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);  in drm_gpuva_link()
2106 	drm_gem_gpuva_assert_lock_held(gpuvm, obj);  in drm_gpuva_link()
2188 drm_gpuva_find_first(struct drm_gpuvm *gpuvm,  in drm_gpuva_find_first()  argument
2193 	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last);  in drm_gpuva_find_first()
2206 drm_gpuva_find(struct drm_gpuvm *gpuvm,  in drm_gpuva_find()  argument
2211 	va = drm_gpuva_find_first(gpuvm, addr, range);  in drm_gpuva_find()
2239 drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start)  in drm_gpuva_find_prev()  argument
2241 	if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1))  in drm_gpuva_find_prev()
2244 	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start);  in drm_gpuva_find_prev()
2261 drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end)  in drm_gpuva_find_next()  argument
2263 	if (!drm_gpuvm_range_valid(gpuvm, end, 1))  in drm_gpuva_find_next()
2266 	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1);  in drm_gpuva_find_next()
2280 drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range)  in drm_gpuvm_interval_empty()  argument
2282 	return !drm_gpuva_find_first(gpuvm, addr, range);  in drm_gpuvm_interval_empty()
2296 drm_gpuva_map(struct drm_gpuvm *gpuvm,  in drm_gpuva_map()  argument
2301 	drm_gpuva_insert(gpuvm, va);  in drm_gpuva_map()
2321 	struct drm_gpuvm *gpuvm = va->vm;  in drm_gpuva_remap()  local
2327 		drm_gpuva_insert(gpuvm, prev);  in drm_gpuva_remap()
2332 		drm_gpuva_insert(gpuvm, next);  in drm_gpuva_remap()
2404 __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm,  in __drm_gpuvm_sm_map()  argument
2418 	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))  in __drm_gpuvm_sm_map()
2421 	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {  in __drm_gpuvm_sm_map()
2583 __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm,  in __drm_gpuvm_sm_unmap()  argument
2591 	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))  in __drm_gpuvm_sm_unmap()
2594 	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {  in __drm_gpuvm_sm_unmap()
2670 drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,  in drm_gpuvm_sm_map()  argument
2673 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_sm_map()
2680 	return __drm_gpuvm_sm_map(gpuvm, ops, priv, req, false);  in drm_gpuvm_sm_map()
2712 drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,  in drm_gpuvm_sm_unmap()  argument
2715 	const struct drm_gpuvm_ops *ops = gpuvm->ops;  in drm_gpuvm_sm_unmap()
2721 	return __drm_gpuvm_sm_unmap(gpuvm, ops, priv,  in drm_gpuvm_sm_unmap()
2806 drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm,  in drm_gpuvm_sm_map_exec_lock()  argument
2818 	return __drm_gpuvm_sm_map(gpuvm, &lock_ops, exec, req, false);  in drm_gpuvm_sm_map_exec_lock()
2838 drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec,  in drm_gpuvm_sm_unmap_exec_lock()  argument
2841 	return __drm_gpuvm_sm_unmap(gpuvm, &lock_ops, exec,  in drm_gpuvm_sm_unmap_exec_lock()
2847 gpuva_op_alloc(struct drm_gpuvm *gpuvm)  in gpuva_op_alloc()  argument
2849 	const struct drm_gpuvm_ops *fn = gpuvm->ops;  in gpuva_op_alloc()
2864 gpuva_op_free(struct drm_gpuvm *gpuvm,  in gpuva_op_free()  argument
2867 	const struct drm_gpuvm_ops *fn = gpuvm->ops;  in gpuva_op_free()
2883 	struct drm_gpuvm *gpuvm = args->vm;  in drm_gpuva_sm_step()  local
2887 	op = gpuva_op_alloc(gpuvm);  in drm_gpuva_sm_step()
2926 	gpuva_op_free(gpuvm, op);  in drm_gpuva_sm_step()
2938 __drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,  in __drm_gpuvm_sm_map_ops_create()  argument
2955 	args.vm = gpuvm;  in __drm_gpuvm_sm_map_ops_create()
2958 	ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, req, madvise);  in __drm_gpuvm_sm_map_ops_create()
2965 	drm_gpuva_ops_free(gpuvm, ops);  in __drm_gpuvm_sm_map_ops_create()
2999 drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,  in drm_gpuvm_sm_map_ops_create()  argument
3002 	return __drm_gpuvm_sm_map_ops_create(gpuvm, req, false);  in drm_gpuvm_sm_map_ops_create()
3036 drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm,  in drm_gpuvm_madvise_ops_create()  argument
3039 	return __drm_gpuvm_sm_map_ops_create(gpuvm, req, true);  in drm_gpuvm_madvise_ops_create()
3073 drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,  in drm_gpuvm_sm_unmap_ops_create()  argument
3089 	args.vm = gpuvm;  in drm_gpuvm_sm_unmap_ops_create()
3092 	ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args,  in drm_gpuvm_sm_unmap_ops_create()
3100 	drm_gpuva_ops_free(gpuvm, ops);  in drm_gpuvm_sm_unmap_ops_create()
3124 drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,  in drm_gpuvm_prefetch_ops_create()  argument
3139 	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {  in drm_gpuvm_prefetch_ops_create()
3140 		op = gpuva_op_alloc(gpuvm);  in drm_gpuvm_prefetch_ops_create()
3154 	drm_gpuva_ops_free(gpuvm, ops);  in drm_gpuvm_prefetch_ops_create()
3222 drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,  in drm_gpuva_ops_free()  argument
3236 		gpuva_op_free(gpuvm, op);  in drm_gpuva_ops_free()