37  * for the entire GPU, there are multiple VM page tables active
38  * at any given time.  The VM page tables can contain a mix
42  * Each VM has an ID associated with it and there is a page table
78  * radeon_vm_manager_init - init the vm manager
82  * Init the vm manager (cayman+).
100  * radeon_vm_manager_fini - tear down the vm manager
104  * Tear down the VM manager (cayman+).
120  * radeon_vm_get_bos - add the vm BOs to a validation list
123  * @vm: vm providing the BOs
130 					  struct radeon_vm *vm,  in radeon_vm_get_bos()  argument
136 	list = kvmalloc_array(vm->max_pde_used + 2,  in radeon_vm_get_bos()
141 	/* add the vm page table to the list */  in radeon_vm_get_bos()
142 	list[0].robj = vm->page_directory;  in radeon_vm_get_bos()
149 	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {  in radeon_vm_get_bos()
150 		if (!vm->page_tables[i].bo)  in radeon_vm_get_bos()
153 		list[idx].robj = vm->page_tables[i].bo;  in radeon_vm_get_bos()
168  * @vm: vm to allocate id for
171  * Allocate an id for the vm (cayman+).
177 				       struct radeon_vm *vm, int ring)  in radeon_vm_grab_id()  argument
180 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_grab_id()
193 	/* skip over VMID 0, since it is the system VM */  in radeon_vm_grab_id()
224  * radeon_vm_flush - hardware flush the vm
227  * @vm: vm we want to flush
229  * @updates: last vm update that is waited for
231  * Flush the vm (cayman+).
236 		     struct radeon_vm *vm,  in radeon_vm_flush()  argument
239 	uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);  in radeon_vm_flush()
240 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_flush()
245 		trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id);  in radeon_vm_flush()
256  * radeon_vm_fence - remember fence for vm
259  * @vm: vm we want to fence
262  * Fence the vm (cayman+).
268 		     struct radeon_vm *vm,  in radeon_vm_fence()  argument
271 	unsigned vm_id = vm->ids[fence->ring].id;  in radeon_vm_fence()
276 	radeon_fence_unref(&vm->ids[fence->ring].last_id_use);  in radeon_vm_fence()
277 	vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence);  in radeon_vm_fence()
281  * radeon_vm_bo_find - find the bo_va for a specific vm & bo
283  * @vm: requested vm
286  * Find @bo inside the requested vm (cayman+).
287  * Search inside the @bos vm list for the requested vm
292 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,  in radeon_vm_bo_find()  argument
298 		if (bo_va->vm == vm)  in radeon_vm_bo_find()
306  * radeon_vm_bo_add - add a bo to a specific vm
309  * @vm: requested vm
312  * Add @bo into the requested vm (cayman+).
313  * Add @bo to the list of bos associated with the vm
319 				      struct radeon_vm *vm,  in radeon_vm_bo_add()  argument
328 	bo_va->vm = vm;  in radeon_vm_bo_add()
337 	mutex_lock(&vm->mutex);  in radeon_vm_bo_add()
339 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_add()
432  * radeon_vm_bo_set_addr - set bos virtual address inside a vm
436  * @soffset: requested offset of the buffer in the VM address space
440  * Validate and set the offset requested within the vm address space.
451 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_set_addr()  local
476 	mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
481 		it = interval_tree_iter_first(&vm->va, soffset, eoffset);  in radeon_vm_bo_set_addr()
489 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
500 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
506 		tmp->vm = vm;  in radeon_vm_bo_set_addr()
509 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
510 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
514 		list_add(&tmp->vm_status, &vm->freed);  in radeon_vm_bo_set_addr()
515 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
519 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
522 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_set_addr()
523 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
524 		interval_tree_insert(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
534 	if (eoffset > vm->max_pde_used)  in radeon_vm_bo_set_addr()
535 		vm->max_pde_used = eoffset;  in radeon_vm_bo_set_addr()
543 		if (vm->page_tables[pt_idx].bo)  in radeon_vm_bo_set_addr()
547 		mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
563 		mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
564 		if (vm->page_tables[pt_idx].bo) {  in radeon_vm_bo_set_addr()
566 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
568 			mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
572 		vm->page_tables[pt_idx].addr = 0;  in radeon_vm_bo_set_addr()
573 		vm->page_tables[pt_idx].bo = pt;  in radeon_vm_bo_set_addr()
576 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
630  * @vm: requested vm
639 				    struct radeon_vm *vm)  in radeon_vm_update_page_directory()  argument
641 	struct radeon_bo *pd = vm->page_directory;  in radeon_vm_update_page_directory()
653 	ndw += vm->max_pde_used * 6;  in radeon_vm_update_page_directory()
665 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {  in radeon_vm_update_page_directory()
666 		struct radeon_bo *bo = vm->page_tables[pt_idx].bo;  in radeon_vm_update_page_directory()
673 		if (vm->page_tables[pt_idx].addr == pt)  in radeon_vm_update_page_directory()
675 		vm->page_tables[pt_idx].addr = pt;  in radeon_vm_update_page_directory()
801  * @vm: requested vm
813 				 struct radeon_vm *vm,  in radeon_vm_update_ptes()  argument
826 		struct radeon_bo *pt = vm->page_tables[pt_idx].bo;  in radeon_vm_update_ptes()
875  * @vm: requested vm
884 static void radeon_vm_fence_pts(struct radeon_vm *vm,  in radeon_vm_fence_pts()  argument
894 		radeon_bo_fence(vm->page_tables[i].bo, fence, true);  in radeon_vm_fence_pts()
898  * radeon_vm_bo_update - map a bo into the vm page table
913 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_update()  local
921 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",  in radeon_vm_bo_update()
922 			bo_va->bo, vm);  in radeon_vm_bo_update()
926 	spin_lock(&vm->status_lock);  in radeon_vm_bo_update()
929 			spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
935 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_update()
937 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
1006 			radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use);  in radeon_vm_bo_update()
1009 	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,  in radeon_vm_bo_update()
1026 	radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence);  in radeon_vm_bo_update()
1038  * @vm: requested vm
1046 			  struct radeon_vm *vm)  in radeon_vm_clear_freed()  argument
1051 	spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1052 	while (!list_empty(&vm->freed)) {  in radeon_vm_clear_freed()
1053 		bo_va = list_first_entry(&vm->freed,  in radeon_vm_clear_freed()
1055 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1060 		spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1067 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1076  * @vm: requested vm
1084 			     struct radeon_vm *vm)  in radeon_vm_clear_invalids()  argument
1089 	spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1090 	while (!list_empty(&vm->invalidated)) {  in radeon_vm_clear_invalids()
1091 		bo_va = list_first_entry(&vm->invalidated,  in radeon_vm_clear_invalids()
1093 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1099 		spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1101 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1107  * radeon_vm_bo_rmv - remove a bo to a specific vm
1112  * Remove @bo_va->bo from the requested vm (cayman+).
1119 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_rmv()  local
1123 	mutex_lock(&vm->mutex);  in radeon_vm_bo_rmv()
1125 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_rmv()
1127 	spin_lock(&vm->status_lock);  in radeon_vm_bo_rmv()
1131 		list_add(&bo_va->vm_status, &vm->freed);  in radeon_vm_bo_rmv()
1136 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_rmv()
1138 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_rmv()
1155 		spin_lock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1158 			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);  in radeon_vm_bo_invalidate()
1159 		spin_unlock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1164  * radeon_vm_init - initialize a vm instance
1167  * @vm: requested vm
1169  * Init @vm fields (cayman+).
1171 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_init()  argument
1178 	vm->ib_bo_va = NULL;  in radeon_vm_init()
1180 		vm->ids[i].id = 0;  in radeon_vm_init()
1181 		vm->ids[i].flushed_updates = NULL;  in radeon_vm_init()
1182 		vm->ids[i].last_id_use = NULL;  in radeon_vm_init()
1184 	mutex_init(&vm->mutex);  in radeon_vm_init()
1185 	vm->va = RB_ROOT_CACHED;  in radeon_vm_init()
1186 	spin_lock_init(&vm->status_lock);  in radeon_vm_init()
1187 	INIT_LIST_HEAD(&vm->invalidated);  in radeon_vm_init()
1188 	INIT_LIST_HEAD(&vm->freed);  in radeon_vm_init()
1189 	INIT_LIST_HEAD(&vm->cleared);  in radeon_vm_init()
1196 	vm->page_tables = kzalloc(pts_size, GFP_KERNEL);  in radeon_vm_init()
1197 	if (vm->page_tables == NULL) {  in radeon_vm_init()
1204 			     NULL, &vm->page_directory);  in radeon_vm_init()
1206 		kfree(vm->page_tables);  in radeon_vm_init()
1207 		vm->page_tables = NULL;  in radeon_vm_init()
1210 	r = radeon_vm_clear_bo(rdev, vm->page_directory);  in radeon_vm_init()
1212 		radeon_bo_unref(&vm->page_directory);  in radeon_vm_init()
1213 		vm->page_directory = NULL;  in radeon_vm_init()
1214 		kfree(vm->page_tables);  in radeon_vm_init()
1215 		vm->page_tables = NULL;  in radeon_vm_init()
1223  * radeon_vm_fini - tear down a vm instance
1226  * @vm: requested vm
1228  * Tear down @vm (cayman+).
1229  * Unbind the VM and remove all bos from the vm bo list
1231 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_fini()  argument
1236 	if (!RB_EMPTY_ROOT(&vm->va.rb_root))  in radeon_vm_fini()
1237 		dev_err(rdev->dev, "still active bo inside vm\n");  in radeon_vm_fini()
1240 					     &vm->va.rb_root, it.rb) {  in radeon_vm_fini()
1241 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_fini()
1250 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {  in radeon_vm_fini()
1257 		radeon_bo_unref(&vm->page_tables[i].bo);  in radeon_vm_fini()
1258 	kfree(vm->page_tables);  in radeon_vm_fini()
1260 	radeon_bo_unref(&vm->page_directory);  in radeon_vm_fini()
1263 		radeon_fence_unref(&vm->ids[i].flushed_updates);  in radeon_vm_fini()
1264 		radeon_fence_unref(&vm->ids[i].last_id_use);  in radeon_vm_fini()
1267 	mutex_destroy(&vm->mutex);  in radeon_vm_fini()