Lines Matching +full:page +full:- +full:level
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
31 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
37 unsigned int level; member
41 * amdgpu_vm_pt_level_shift - return the addr shift for each level
44 * @level: VMPT level
47 * The number of bits the pfn needs to be right shifted for a level.
50 unsigned int level) in amdgpu_vm_pt_level_shift() argument
52 switch (level) { in amdgpu_vm_pt_level_shift()
56 return 9 * (AMDGPU_VM_PDB0 - level) + in amdgpu_vm_pt_level_shift()
57 adev->vm_manager.block_size; in amdgpu_vm_pt_level_shift()
66 * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT
69 * @level: VMPT level
72 * The number of entries in a page directory or page table.
75 unsigned int level) in amdgpu_vm_pt_num_entries() argument
79 shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); in amdgpu_vm_pt_num_entries()
80 if (level == adev->vm_manager.root_level) in amdgpu_vm_pt_num_entries()
82 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) in amdgpu_vm_pt_num_entries()
84 else if (level != AMDGPU_VM_PTB) in amdgpu_vm_pt_num_entries()
88 /* For the page tables on the leaves */ in amdgpu_vm_pt_num_entries()
93 * amdgpu_vm_pt_num_ats_entries - return the number of ATS entries in the root PD
98 * The number of entries in the root page directory which needs the ATS setting.
104 shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); in amdgpu_vm_pt_num_ats_entries()
109 * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT
112 * @level: VMPT level
118 unsigned int level) in amdgpu_vm_pt_entries_mask() argument
120 if (level <= adev->vm_manager.root_level) in amdgpu_vm_pt_entries_mask()
122 else if (level != AMDGPU_VM_PTB) in amdgpu_vm_pt_entries_mask()
125 return AMDGPU_VM_PTE_COUNT(adev) - 1; in amdgpu_vm_pt_entries_mask()
129 * amdgpu_vm_pt_size - returns the size of the page table in bytes
132 * @level: VMPT level
135 * The size of the BO for a page directory or page table in bytes.
138 unsigned int level) in amdgpu_vm_pt_size() argument
140 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8); in amdgpu_vm_pt_size()
144 * amdgpu_vm_pt_parent - get the parent page directory
146 * @pt: child page table
148 * Helper to get the parent entry for the child page table. NULL if we are at
149 * the root page directory.
154 struct amdgpu_bo *parent = pt->bo->parent; in amdgpu_vm_pt_parent()
159 return parent->vm_bo; in amdgpu_vm_pt_parent()
163 * amdgpu_vm_pt_start - start PD/PT walk
176 cursor->pfn = start; in amdgpu_vm_pt_start()
177 cursor->parent = NULL; in amdgpu_vm_pt_start()
178 cursor->entry = &vm->root; in amdgpu_vm_pt_start()
179 cursor->level = adev->vm_manager.root_level; in amdgpu_vm_pt_start()
183 * amdgpu_vm_pt_descendant - go to child node
197 if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || in amdgpu_vm_pt_descendant()
198 !cursor->entry->bo) in amdgpu_vm_pt_descendant()
201 mask = amdgpu_vm_pt_entries_mask(adev, cursor->level); in amdgpu_vm_pt_descendant()
202 shift = amdgpu_vm_pt_level_shift(adev, cursor->level); in amdgpu_vm_pt_descendant()
204 ++cursor->level; in amdgpu_vm_pt_descendant()
205 idx = (cursor->pfn >> shift) & mask; in amdgpu_vm_pt_descendant()
206 cursor->parent = cursor->entry; in amdgpu_vm_pt_descendant()
207 cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; in amdgpu_vm_pt_descendant()
212 * amdgpu_vm_pt_sibling - go to sibling node
229 if (!cursor->parent) in amdgpu_vm_pt_sibling()
233 shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1); in amdgpu_vm_pt_sibling()
234 num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1); in amdgpu_vm_pt_sibling()
235 parent = to_amdgpu_bo_vm(cursor->parent->bo); in amdgpu_vm_pt_sibling()
237 if (cursor->entry == &parent->entries[num_entries - 1]) in amdgpu_vm_pt_sibling()
240 cursor->pfn += 1ULL << shift; in amdgpu_vm_pt_sibling()
241 cursor->pfn &= ~((1ULL << shift) - 1); in amdgpu_vm_pt_sibling()
242 ++cursor->entry; in amdgpu_vm_pt_sibling()
247 * amdgpu_vm_pt_ancestor - go to parent node
257 if (!cursor->parent) in amdgpu_vm_pt_ancestor()
260 --cursor->level; in amdgpu_vm_pt_ancestor()
261 cursor->entry = cursor->parent; in amdgpu_vm_pt_ancestor()
262 cursor->parent = amdgpu_vm_pt_parent(cursor->parent); in amdgpu_vm_pt_ancestor()
267 * amdgpu_vm_pt_next - get next PD/PT in hieratchy
285 cursor->pfn = ~0ll; in amdgpu_vm_pt_next()
292 * amdgpu_vm_pt_first_dfs - start a deep first search
316 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
327 return entry && (!start || entry != start->entry); in amdgpu_vm_pt_continue_dfs()
331 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
341 if (!cursor->entry) in amdgpu_vm_pt_next_dfs()
344 if (!cursor->parent) in amdgpu_vm_pt_next_dfs()
345 cursor->entry = NULL; in amdgpu_vm_pt_next_dfs()
354 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
363 * amdgpu_vm_pt_clear - initially clear the PDs/PTs
378 unsigned int level = adev->vm_manager.root_level; in amdgpu_vm_pt_clear() local
381 struct amdgpu_bo *ancestor = &vmbo->bo; in amdgpu_vm_pt_clear()
383 struct amdgpu_bo *bo = &vmbo->bo; in amdgpu_vm_pt_clear()
388 if (ancestor->parent) { in amdgpu_vm_pt_clear()
389 ++level; in amdgpu_vm_pt_clear()
390 while (ancestor->parent->parent) { in amdgpu_vm_pt_clear()
391 ++level; in amdgpu_vm_pt_clear()
392 ancestor = ancestor->parent; in amdgpu_vm_pt_clear()
397 if (!vm->pte_support_ats) { in amdgpu_vm_pt_clear()
400 } else if (!bo->parent) { in amdgpu_vm_pt_clear()
403 entries -= ats_entries; in amdgpu_vm_pt_clear()
408 pt = ancestor->vm_bo; in amdgpu_vm_pt_clear()
410 if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >= in amdgpu_vm_pt_clear()
419 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in amdgpu_vm_pt_clear()
423 if (vmbo->shadow) { in amdgpu_vm_pt_clear()
424 struct amdgpu_bo *shadow = vmbo->shadow; in amdgpu_vm_pt_clear()
426 r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx); in amdgpu_vm_pt_clear()
432 return -ENODEV; in amdgpu_vm_pt_clear()
434 r = vm->update_funcs->map_table(vmbo); in amdgpu_vm_pt_clear()
443 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); in amdgpu_vm_pt_clear()
452 if (level != AMDGPU_VM_PTB) { in amdgpu_vm_pt_clear()
455 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); in amdgpu_vm_pt_clear()
458 r = vm->update_funcs->update(¶ms, vmbo, addr, 0, in amdgpu_vm_pt_clear()
469 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_vm_pt_clear()
470 if (level != AMDGPU_VM_PTB) { in amdgpu_vm_pt_clear()
473 amdgpu_gmc_get_vm_pde(adev, level, in amdgpu_vm_pt_clear()
481 r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries, in amdgpu_vm_pt_clear()
487 r = vm->update_funcs->commit(¶ms, NULL); in amdgpu_vm_pt_clear()
494 * amdgpu_vm_pt_create - create bo for PD/PT
498 * @level: the page table level
504 int level, bool immediate, struct amdgpu_bo_vm **vmbo, in amdgpu_vm_pt_create() argument
515 bp.size = amdgpu_vm_pt_size(adev, level); in amdgpu_vm_pt_create()
518 if (!adev->gmc.is_app_apu) in amdgpu_vm_pt_create()
527 if (level < AMDGPU_VM_PTB) in amdgpu_vm_pt_create()
528 num_entries = amdgpu_vm_pt_num_entries(adev, level); in amdgpu_vm_pt_create()
534 if (vm->use_cpu_for_update) in amdgpu_vm_pt_create()
541 if (vm->root.bo) in amdgpu_vm_pt_create()
542 bp.resv = vm->root.bo->tbo.base.resv; in amdgpu_vm_pt_create()
548 bo = &(*vmbo)->bo; in amdgpu_vm_pt_create()
549 if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) { in amdgpu_vm_pt_create()
550 (*vmbo)->shadow = NULL; in amdgpu_vm_pt_create()
555 WARN_ON(dma_resv_lock(bo->tbo.base.resv, in amdgpu_vm_pt_create()
559 bp.size = amdgpu_vm_pt_size(adev, level); in amdgpu_vm_pt_create()
563 bp.resv = bo->tbo.base.resv; in amdgpu_vm_pt_create()
567 r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow); in amdgpu_vm_pt_create()
570 dma_resv_unlock(bo->tbo.base.resv); in amdgpu_vm_pt_create()
583 * amdgpu_vm_pt_alloc - Allocate a specific page table
586 * @vm: VM to allocate page tables for
587 * @cursor: Which page table to allocate
590 * Make sure a specific page table or directory is allocated.
593 * 1 if page table needed to be allocated, 0 if page table was already
601 struct amdgpu_vm_bo_base *entry = cursor->entry; in amdgpu_vm_pt_alloc()
606 if (entry->bo) in amdgpu_vm_pt_alloc()
610 r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt, in amdgpu_vm_pt_alloc()
611 vm->root.bo->xcp_id); in amdgpu_vm_pt_alloc()
619 pt_bo = &pt->bo; in amdgpu_vm_pt_alloc()
620 pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); in amdgpu_vm_pt_alloc()
629 amdgpu_bo_unref(&pt->shadow); in amdgpu_vm_pt_alloc()
635 * amdgpu_vm_pt_free - free one PD/PT
643 if (!entry->bo) in amdgpu_vm_pt_free()
646 entry->bo->vm_bo = NULL; in amdgpu_vm_pt_free()
647 shadow = amdgpu_bo_shadowed(entry->bo); in amdgpu_vm_pt_free()
649 ttm_bo_set_bulk_move(&shadow->tbo, NULL); in amdgpu_vm_pt_free()
652 ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); in amdgpu_vm_pt_free()
654 spin_lock(&entry->vm->status_lock); in amdgpu_vm_pt_free()
655 list_del(&entry->vm_status); in amdgpu_vm_pt_free()
656 spin_unlock(&entry->vm->status_lock); in amdgpu_vm_pt_free()
657 amdgpu_bo_unref(&entry->bo); in amdgpu_vm_pt_free()
668 spin_lock(&vm->status_lock); in amdgpu_vm_pt_free_work()
669 list_splice_init(&vm->pt_freed, &pt_freed); in amdgpu_vm_pt_free_work()
670 spin_unlock(&vm->status_lock); in amdgpu_vm_pt_free_work()
672 /* flush_work in amdgpu_vm_fini ensure vm->root.bo is valid. */ in amdgpu_vm_pt_free_work()
673 amdgpu_bo_reserve(vm->root.bo, true); in amdgpu_vm_pt_free_work()
678 amdgpu_bo_unreserve(vm->root.bo); in amdgpu_vm_pt_free_work()
682 * amdgpu_vm_pt_free_dfs - free PD/PT levels
689 * Free the page directory or page table level and all sub levels.
700 spin_lock(&vm->status_lock); in amdgpu_vm_pt_free_dfs()
702 list_move(&entry->vm_status, &vm->pt_freed); in amdgpu_vm_pt_free_dfs()
705 list_move(&start->entry->vm_status, &vm->pt_freed); in amdgpu_vm_pt_free_dfs()
706 spin_unlock(&vm->status_lock); in amdgpu_vm_pt_free_dfs()
707 schedule_work(&vm->pt_free_work); in amdgpu_vm_pt_free_dfs()
715 amdgpu_vm_pt_free(start->entry); in amdgpu_vm_pt_free_dfs()
719 * amdgpu_vm_pt_free_root - free root PD
723 * Free the root page directory and everything below it.
731 * amdgpu_vm_pt_is_root_clean - check if a root PD is clean
737 * it means there are page table creating and filling, and is no a clean
746 enum amdgpu_vm_level root = adev->vm_manager.root_level; in amdgpu_vm_pt_is_root_clean()
751 if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo) in amdgpu_vm_pt_is_root_clean()
758 * amdgpu_vm_pde_update - update a single level in the hierarchy
769 struct amdgpu_bo *bo = parent->bo, *pbo; in amdgpu_vm_pde_update()
770 struct amdgpu_vm *vm = params->vm; in amdgpu_vm_pde_update()
772 unsigned int level; in amdgpu_vm_pde_update() local
774 for (level = 0, pbo = bo->parent; pbo; ++level) in amdgpu_vm_pde_update()
775 pbo = pbo->parent; in amdgpu_vm_pde_update()
777 level += params->adev->vm_manager.root_level; in amdgpu_vm_pde_update()
778 amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags); in amdgpu_vm_pde_update()
779 pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8; in amdgpu_vm_pde_update()
780 return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt, in amdgpu_vm_pde_update()
785 * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags
790 * Update PTE no-retry flags when TF is enabled.
796 * Update no-retry flags with the corresponding TF in amdgpu_vm_pte_update_noretry_flags()
797 * no-retry combination. in amdgpu_vm_pte_update_noretry_flags()
801 *flags |= adev->gmc.noretry_flags; in amdgpu_vm_pte_update_noretry_flags()
806 * amdgpu_vm_pte_update_flags - figure out flags for PTE updates
808 * Make sure to set the right flags for the PTEs at the desired level.
812 unsigned int level, in amdgpu_vm_pte_update_flags() argument
817 struct amdgpu_device *adev = params->adev; in amdgpu_vm_pte_update_flags()
819 if (level != AMDGPU_VM_PTB) { in amdgpu_vm_pte_update_flags()
821 amdgpu_gmc_get_vm_pde(adev, level, &addr, &flags); in amdgpu_vm_pte_update_flags()
823 } else if (adev->asic_type >= CHIP_VEGA10 && in amdgpu_vm_pte_update_flags()
832 * Update no-retry flags to use the no-retry flag combination in amdgpu_vm_pte_update_flags()
838 if (level == AMDGPU_VM_PTB) in amdgpu_vm_pte_update_flags()
845 if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) && in amdgpu_vm_pte_update_flags()
846 adev->gmc.gmc_funcs->override_vm_pte_flags && in amdgpu_vm_pte_update_flags()
847 num_possible_nodes() > 1 && !params->pages_addr && params->allow_override) in amdgpu_vm_pte_update_flags()
848 amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags); in amdgpu_vm_pte_update_flags()
850 params->vm->update_funcs->update(params, pt, pe, addr, count, incr, in amdgpu_vm_pte_update_flags()
855 * amdgpu_vm_pte_fragment - get fragment for PTEs
872 * field in the PTE. When this field is set to a non-zero value, page in amdgpu_vm_pte_fragment()
893 if (params->adev->asic_type < CHIP_VEGA10) in amdgpu_vm_pte_fragment()
894 max_frag = params->adev->vm_manager.fragment_size; in amdgpu_vm_pte_fragment()
899 if (params->pages_addr) { in amdgpu_vm_pte_fragment()
906 *frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1); in amdgpu_vm_pte_fragment()
909 *frag_end = end & ~((1ULL << max_frag) - 1); in amdgpu_vm_pte_fragment()
916 * amdgpu_vm_ptes_update - make sure that page tables are valid
924 * Update the page tables in the range @start - @end.
927 * 0 for success, -EINVAL for failure.
933 struct amdgpu_device *adev = params->adev; in amdgpu_vm_ptes_update()
944 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); in amdgpu_vm_ptes_update()
950 if (!params->unlocked) { in amdgpu_vm_ptes_update()
951 /* make sure that the page tables covering the in amdgpu_vm_ptes_update()
954 r = amdgpu_vm_pt_alloc(params->adev, params->vm, in amdgpu_vm_ptes_update()
955 &cursor, params->immediate); in amdgpu_vm_ptes_update()
960 shift = amdgpu_vm_pt_level_shift(adev, cursor.level); in amdgpu_vm_ptes_update()
961 parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1); in amdgpu_vm_ptes_update()
962 if (params->unlocked) { in amdgpu_vm_ptes_update()
966 } else if (adev->asic_type < CHIP_VEGA10 && in amdgpu_vm_ptes_update()
968 /* No huge page support before GMC v9 */ in amdgpu_vm_ptes_update()
969 if (cursor.level != AMDGPU_VM_PTB) { in amdgpu_vm_ptes_update()
971 return -ENOENT; in amdgpu_vm_ptes_update()
975 /* We can't use this level when the fragment size is in amdgpu_vm_ptes_update()
983 * shift we should go up one level and check it again. in amdgpu_vm_ptes_update()
986 return -EINVAL; in amdgpu_vm_ptes_update()
990 pt = cursor.entry->bo; in amdgpu_vm_ptes_update()
994 return -ENOENT; in amdgpu_vm_ptes_update()
997 * level. in amdgpu_vm_ptes_update()
1000 return -EINVAL; in amdgpu_vm_ptes_update()
1002 pt = cursor.entry->bo; in amdgpu_vm_ptes_update()
1010 mask = amdgpu_vm_pt_entries_mask(adev, cursor.level); in amdgpu_vm_ptes_update()
1013 entry_end += cursor.pfn & ~(entry_end - 1); in amdgpu_vm_ptes_update()
1017 struct amdgpu_vm *vm = params->vm; in amdgpu_vm_ptes_update()
1019 unsigned int nptes = (upd_end - frag_start) >> shift; in amdgpu_vm_ptes_update()
1022 /* This can happen when we set higher level PDs to in amdgpu_vm_ptes_update()
1030 vm->task_info.tgid, in amdgpu_vm_ptes_update()
1031 vm->immediate.fence_context); in amdgpu_vm_ptes_update()
1033 cursor.level, pe_start, dst, in amdgpu_vm_ptes_update()
1053 * This is the maximum you can free, because all other page tables are not in amdgpu_vm_ptes_update()
1058 if (cursor.entry->bo) { in amdgpu_vm_ptes_update()
1059 params->table_freed = true; in amdgpu_vm_ptes_update()
1060 amdgpu_vm_pt_free_dfs(adev, params->vm, in amdgpu_vm_ptes_update()
1062 params->unlocked); in amdgpu_vm_ptes_update()
1068 /* or just move on to the next on the same level. */ in amdgpu_vm_ptes_update()
1077 * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
1081 * make root page directory and everything below it cpu accessible.
1093 if (entry->bo) { in amdgpu_vm_pt_map_tables()
1094 bo = to_amdgpu_bo_vm(entry->bo); in amdgpu_vm_pt_map_tables()
1095 r = vm->update_funcs->map_table(bo); in amdgpu_vm_pt_map_tables()