Lines Matching full:vm
90 struct drm_i915_private *i915 = ppgtt->vm.i915; in gen8_ppgtt_notify_vgt()
91 struct intel_uncore *uncore = ppgtt->vm.gt->uncore; in gen8_ppgtt_notify_vgt()
102 if (i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_notify_vgt()
180 static unsigned int gen8_pd_top_count(const struct i915_address_space *vm) in gen8_pd_top_count() argument
182 unsigned int shift = __gen8_pte_shift(vm->top); in gen8_pd_top_count()
184 return (vm->total + (1ull << shift) - 1) >> shift; in gen8_pd_top_count()
188 gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx) in gen8_pdp_for_page_index() argument
190 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); in gen8_pdp_for_page_index()
192 if (vm->top == 2) in gen8_pdp_for_page_index()
195 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top)); in gen8_pdp_for_page_index()
199 gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr) in gen8_pdp_for_page_address() argument
201 return gen8_pdp_for_page_index(vm, addr >> GEN8_PTE_SHIFT); in gen8_pdp_for_page_address()
204 static void __gen8_ppgtt_cleanup(struct i915_address_space *vm, in __gen8_ppgtt_cleanup() argument
215 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1); in __gen8_ppgtt_cleanup()
219 free_px(vm, &pd->pt, lvl); in __gen8_ppgtt_cleanup()
222 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) in gen8_ppgtt_cleanup() argument
224 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); in gen8_ppgtt_cleanup()
226 if (vm->rsvd.obj) in gen8_ppgtt_cleanup()
227 i915_gem_object_put(vm->rsvd.obj); in gen8_ppgtt_cleanup()
229 if (intel_vgpu_active(vm->i915)) in gen8_ppgtt_cleanup()
233 __gen8_ppgtt_cleanup(vm, ppgtt->pd, in gen8_ppgtt_cleanup()
234 gen8_pd_top_count(vm), vm->top); in gen8_ppgtt_cleanup()
236 free_scratch(vm); in gen8_ppgtt_cleanup()
239 static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, in __gen8_ppgtt_clear() argument
243 const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; in __gen8_ppgtt_clear()
246 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_clear()
250 __func__, vm, lvl + 1, start, end, in __gen8_ppgtt_clear()
260 __func__, vm, lvl + 1, idx, start, end); in __gen8_ppgtt_clear()
262 __gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl); in __gen8_ppgtt_clear()
268 start = __gen8_ppgtt_clear(vm, as_pd(pt), in __gen8_ppgtt_clear()
278 __func__, vm, lvl, start, end, in __gen8_ppgtt_clear()
293 vm->scratch[0]->encode, in __gen8_ppgtt_clear()
301 free_px(vm, pt, lvl); in __gen8_ppgtt_clear()
307 static void gen8_ppgtt_clear(struct i915_address_space *vm, in gen8_ppgtt_clear() argument
312 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_clear()
318 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_clear()
319 start, start + length, vm->top); in gen8_ppgtt_clear()
322 static void __gen8_ppgtt_alloc(struct i915_address_space * const vm, in __gen8_ppgtt_alloc() argument
329 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_alloc()
333 __func__, vm, lvl + 1, *start, end, in __gen8_ppgtt_alloc()
346 __func__, vm, lvl + 1, idx); in __gen8_ppgtt_alloc()
351 fill_px(pt, vm->scratch[lvl]->encode); in __gen8_ppgtt_alloc()
367 __gen8_ppgtt_alloc(vm, stash, in __gen8_ppgtt_alloc()
377 __func__, vm, lvl, *start, end, in __gen8_ppgtt_alloc()
390 static void gen8_ppgtt_alloc(struct i915_address_space *vm, in gen8_ppgtt_alloc() argument
396 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_alloc()
402 __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_alloc()
403 &start, start + length, vm->top); in gen8_ppgtt_alloc()
406 static void __gen8_ppgtt_foreach(struct i915_address_space *vm, in __gen8_ppgtt_foreach() argument
409 void (*fn)(struct i915_address_space *vm, in __gen8_ppgtt_foreach() argument
426 __gen8_ppgtt_foreach(vm, as_pd(pt), start, end, lvl, in __gen8_ppgtt_foreach()
429 fn(vm, pt, data); in __gen8_ppgtt_foreach()
439 static void gen8_ppgtt_foreach(struct i915_address_space *vm, in gen8_ppgtt_foreach() argument
441 void (*fn)(struct i915_address_space *vm, in gen8_ppgtt_foreach() argument
449 __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_foreach()
450 &start, start + length, vm->top, in gen8_ppgtt_foreach()
463 const gen8_pte_t pte_encode = ppgtt->vm.pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_pte()
503 xehp_ppgtt_insert_huge(struct i915_address_space *vm, in xehp_ppgtt_insert_huge() argument
509 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in xehp_ppgtt_insert_huge()
514 GEM_BUG_ON(!i915_vm_is_4lvl(vm)); in xehp_ppgtt_insert_huge()
518 gen8_pdp_for_page_address(vm, start); in xehp_ppgtt_insert_huge()
611 static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, in gen8_ppgtt_insert_huge() argument
617 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); in gen8_ppgtt_insert_huge()
621 GEM_BUG_ON(!i915_vm_is_4lvl(vm)); in gen8_ppgtt_insert_huge()
625 gen8_pdp_for_page_address(vm, start); in gen8_ppgtt_insert_huge()
700 (i915_vm_has_scratch_64K(vm) && in gen8_ppgtt_insert_huge()
718 if (I915_SELFTEST_ONLY(vm->scrub_64K)) { in gen8_ppgtt_insert_huge()
721 encode = vm->scratch[0]->encode; in gen8_ppgtt_insert_huge()
735 static void gen8_ppgtt_insert(struct i915_address_space *vm, in gen8_ppgtt_insert() argument
740 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); in gen8_ppgtt_insert()
744 if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 55)) in gen8_ppgtt_insert()
745 xehp_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); in gen8_ppgtt_insert()
747 gen8_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); in gen8_ppgtt_insert()
753 gen8_pdp_for_page_index(vm, idx); in gen8_ppgtt_insert()
763 static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, in gen8_ppgtt_insert_entry() argument
771 gen8_pdp_for_page_index(vm, idx); in gen8_ppgtt_insert_entry()
780 vaddr[gen8_pd_index(idx, 0)] = vm->pte_encode(addr, pat_index, flags); in gen8_ppgtt_insert_entry()
784 static void xehp_ppgtt_insert_entry_lm(struct i915_address_space *vm, in xehp_ppgtt_insert_entry_lm() argument
792 gen8_pdp_for_page_index(vm, idx); in xehp_ppgtt_insert_entry_lm()
810 vaddr[gen8_pd_index(idx, 0) / 16] = vm->pte_encode(addr, pat_index, flags); in xehp_ppgtt_insert_entry_lm()
813 static void xehp_ppgtt_insert_entry(struct i915_address_space *vm, in xehp_ppgtt_insert_entry() argument
820 return xehp_ppgtt_insert_entry_lm(vm, addr, offset, in xehp_ppgtt_insert_entry()
823 return gen8_ppgtt_insert_entry(vm, addr, offset, pat_index, flags); in xehp_ppgtt_insert_entry()
826 static int gen8_init_scratch(struct i915_address_space *vm) in gen8_init_scratch() argument
834 * we can reuse it for all vm, keeping contexts and processes separate. in gen8_init_scratch()
836 if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) { in gen8_init_scratch()
837 struct i915_address_space *clone = vm->gt->vm; in gen8_init_scratch()
841 vm->scratch_order = clone->scratch_order; in gen8_init_scratch()
842 for (i = 0; i <= vm->top; i++) in gen8_init_scratch()
843 vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); in gen8_init_scratch()
848 ret = setup_scratch_page(vm); in gen8_init_scratch()
852 pte_flags = vm->has_read_only; in gen8_init_scratch()
853 if (i915_gem_object_is_lmem(vm->scratch[0])) in gen8_init_scratch()
856 vm->scratch[0]->encode = in gen8_init_scratch()
857 vm->pte_encode(px_dma(vm->scratch[0]), in gen8_init_scratch()
858 i915_gem_get_pat_index(vm->i915, in gen8_init_scratch()
862 for (i = 1; i <= vm->top; i++) { in gen8_init_scratch()
865 obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_init_scratch()
871 ret = map_pt_dma(vm, obj); in gen8_init_scratch()
877 fill_px(obj, vm->scratch[i - 1]->encode); in gen8_init_scratch()
880 vm->scratch[i] = obj; in gen8_init_scratch()
887 i915_gem_object_put(vm->scratch[i]); in gen8_init_scratch()
888 vm->scratch[0] = NULL; in gen8_init_scratch()
894 struct i915_address_space *vm = &ppgtt->vm; in gen8_preallocate_top_level_pdp() local
898 GEM_BUG_ON(vm->top != 2); in gen8_preallocate_top_level_pdp()
899 GEM_BUG_ON(gen8_pd_top_count(vm) != GEN8_3LVL_PDPES); in gen8_preallocate_top_level_pdp()
905 pde = alloc_pd(vm); in gen8_preallocate_top_level_pdp()
909 err = map_pt_dma(vm, pde->pt.base); in gen8_preallocate_top_level_pdp()
911 free_pd(vm, pde); in gen8_preallocate_top_level_pdp()
915 fill_px(pde, vm->scratch[1]->encode); in gen8_preallocate_top_level_pdp()
925 gen8_alloc_top_pd(struct i915_address_space *vm) in gen8_alloc_top_pd() argument
927 const unsigned int count = gen8_pd_top_count(vm); in gen8_alloc_top_pd()
937 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_alloc_top_pd()
944 err = map_pt_dma(vm, pd->pt.base); in gen8_alloc_top_pd()
948 fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); in gen8_alloc_top_pd()
953 free_pd(vm, pd); in gen8_alloc_top_pd()
957 static int gen8_init_rsvd(struct i915_address_space *vm) in gen8_init_rsvd() argument
959 struct drm_i915_private *i915 = vm->i915; in gen8_init_rsvd()
964 if (!intel_gt_needs_wa_16018031267(vm->gt)) in gen8_init_rsvd()
976 vma = i915_vma_instance(obj, vm, NULL); in gen8_init_rsvd()
986 vm->rsvd.vma = i915_vma_make_unshrinkable(vma); in gen8_init_rsvd()
987 vm->rsvd.obj = obj; in gen8_init_rsvd()
988 vm->total -= vma->node.size; in gen8_init_rsvd()
1014 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; in gen8_ppgtt_create()
1015 ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); in gen8_ppgtt_create()
1025 ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12); in gen8_ppgtt_create()
1028 ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; in gen8_ppgtt_create()
1030 ppgtt->vm.alloc_pt_dma = alloc_pt_dma; in gen8_ppgtt_create()
1038 ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; in gen8_ppgtt_create()
1041 ppgtt->vm.pte_encode = gen12_pte_encode; in gen8_ppgtt_create()
1043 ppgtt->vm.pte_encode = gen8_pte_encode; in gen8_ppgtt_create()
1045 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; in gen8_ppgtt_create()
1046 ppgtt->vm.insert_entries = gen8_ppgtt_insert; in gen8_ppgtt_create()
1048 ppgtt->vm.insert_page = xehp_ppgtt_insert_entry; in gen8_ppgtt_create()
1050 ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; in gen8_ppgtt_create()
1051 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; in gen8_ppgtt_create()
1052 ppgtt->vm.clear_range = gen8_ppgtt_clear; in gen8_ppgtt_create()
1053 ppgtt->vm.foreach = gen8_ppgtt_foreach; in gen8_ppgtt_create()
1054 ppgtt->vm.cleanup = gen8_ppgtt_cleanup; in gen8_ppgtt_create()
1056 err = gen8_init_scratch(&ppgtt->vm); in gen8_ppgtt_create()
1060 pd = gen8_alloc_top_pd(&ppgtt->vm); in gen8_ppgtt_create()
1067 if (!i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_create()
1076 err = gen8_init_rsvd(&ppgtt->vm); in gen8_ppgtt_create()
1083 i915_vm_put(&ppgtt->vm); in gen8_ppgtt_create()