Lines Matching +full:parent +full:- +full:locked
1 // SPDX-License-Identifier: MIT
23 /** @children: Array of page-table child nodes */
28 #define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
29 #define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
38 #define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
47 return container_of(pt_dir->children[index], struct xe_pt, base); in xe_pt_entry()
54 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; in __xe_pt_empty_pte()
55 u8 id = tile->id; in __xe_pt_empty_pte()
61 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, in __xe_pt_empty_pte()
64 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | in __xe_pt_empty_pte()
70 if (pt->level) in xe_pt_free()
77 * xe_pt_create() - Create a page-table.
80 * @level: The page-table level.
83 * create the corresponding page-table bo, but don't initialize it. If the
84 * level is grater than zero, then it's assumed to be a directory page-
101 pt = (dir) ? &dir->pt : NULL; in xe_pt_create()
106 return ERR_PTR(-ENOMEM); in xe_pt_create()
108 pt->level = level; in xe_pt_create()
109 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, in xe_pt_create()
120 pt->bo = bo; in xe_pt_create()
121 pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; in xe_pt_create()
123 if (vm->xef) in xe_pt_create()
124 xe_drm_client_add_bo(vm->xef->client, pt->bo); in xe_pt_create()
135 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
141 * Populate the page-table bo of @pt with entries pointing into the tile's
142 * scratch page-table tree if any. Otherwise populate with zeros.
147 struct iosys_map *map = &pt->bo->vmap; in xe_pt_populate_empty()
156 xe_map_memset(vm->xe, map, 0, 0, SZ_4K); in xe_pt_populate_empty()
158 empty = __xe_pt_empty_pte(tile, vm, pt->level); in xe_pt_populate_empty()
160 xe_pt_write(vm->xe, map, i, empty); in xe_pt_populate_empty()
165 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
166 * a page-table at a certain level.
169 * Return: The ilog2 value of the size of the address range of a page-table
178 * xe_pt_destroy() - Destroy a page-table tree.
179 * @pt: The root of the page-table tree to destroy.
184 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
194 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); in xe_pt_destroy()
195 xe_bo_unpin(pt->bo); in xe_pt_destroy()
196 xe_bo_put_deferred(pt->bo, deferred); in xe_pt_destroy()
198 if (pt->level > 0 && pt->num_live) { in xe_pt_destroy()
213 * Below we use the term "page-table" for both page-directories, containing
214 * pointers to lower level page-directories or page-tables, and level 0
215 * page-tables that contain only page-table-entries pointing to memory pages.
217 * When inserting an address range in an already existing page-table tree
218 * there will typically be a set of page-tables that are shared with other
220 * The set of shared page-tables can be at most two per level,
222 * page-tables may still be in use by the gpu for other mappings. Therefore
225 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
226 * added in a separate commit step. CPU-data is committed while still under the
230 * For non-shared page-tables (and, in fact, for shared ones that aren't
231 * existing at the time of staging), we add the data in-place without the
232 * special update structures. This private part of the page-table tree will
233 * remain disconnected from the vm page-table tree until data is committed to
238 /** @update: The update structure we're building for this parent. */
240 /** @parent: The parent. Used to detect a parent change. */
241 struct xe_pt *parent; member
242 /** @preexisting: Whether the parent was pre-existing or allocated */
272 /** @va_curs_start: The Virtual address coresponding to @curs->start */
298 xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, in xe_pt_new_shared() argument
301 struct xe_pt_update *upd = &wupd->updates[parent->level]; in xe_pt_new_shared()
307 * new parent and page-directory, the old one is complete, and in xe_pt_new_shared()
309 * @wupd->entries in xe_pt_new_shared()
311 if (likely(upd->parent == parent)) in xe_pt_new_shared()
314 upd->parent = parent; in xe_pt_new_shared()
315 upd->preexisting = true; in xe_pt_new_shared()
317 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) in xe_pt_new_shared()
318 return -EINVAL; in xe_pt_new_shared()
320 entry = wupd->entries + wupd->num_used_entries++; in xe_pt_new_shared()
321 upd->update = entry; in xe_pt_new_shared()
322 entry->ofs = offset; in xe_pt_new_shared()
323 entry->pt_bo = parent->bo; in xe_pt_new_shared()
324 entry->pt = parent; in xe_pt_new_shared()
325 entry->flags = 0; in xe_pt_new_shared()
326 entry->qwords = 0; in xe_pt_new_shared()
329 entry->pt_entries = kmalloc_array(XE_PDES, in xe_pt_new_shared()
330 sizeof(*entry->pt_entries), in xe_pt_new_shared()
332 if (!entry->pt_entries) in xe_pt_new_shared()
333 return -ENOMEM; in xe_pt_new_shared()
342 * non-pre-existing pagetable with leaf ptes.
345 xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, in xe_pt_insert_entry() argument
348 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; in xe_pt_insert_entry()
350 &xe_walk->wupd.updates[xe_child->level] : NULL; in xe_pt_insert_entry()
353 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); in xe_pt_insert_entry()
362 child_upd->update = NULL; in xe_pt_insert_entry()
363 child_upd->parent = xe_child; in xe_pt_insert_entry()
364 child_upd->preexisting = false; in xe_pt_insert_entry()
367 if (likely(!upd->preexisting)) { in xe_pt_insert_entry()
368 /* Continue building a non-connected subtree. */ in xe_pt_insert_entry()
369 struct iosys_map *map = &parent->bo->vmap; in xe_pt_insert_entry()
372 parent->base.children[offset] = &xe_child->base; in xe_pt_insert_entry()
374 xe_pt_write(xe_walk->vm->xe, map, offset, pte); in xe_pt_insert_entry()
375 parent->num_live++; in xe_pt_insert_entry()
379 struct xe_vm_pgtable_update *entry = upd->update; in xe_pt_insert_entry()
381 idx = offset - entry->ofs; in xe_pt_insert_entry()
382 entry->pt_entries[idx].pt = xe_child; in xe_pt_insert_entry()
383 entry->pt_entries[idx].pte = pte; in xe_pt_insert_entry()
384 entry->qwords++; in xe_pt_insert_entry()
399 if (!xe_pt_covers(addr, next, level, &xe_walk->base)) in xe_pt_hugepte_possible()
403 if (next - xe_walk->va_curs_start > xe_walk->curs->size) in xe_pt_hugepte_possible()
407 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_hugepte_possible()
411 size = next - addr; in xe_pt_hugepte_possible()
412 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); in xe_pt_hugepte_possible()
424 struct xe_res_cursor curs = *xe_walk->curs; in xe_pt_scan_64K()
429 if (next > xe_walk->l0_end_addr) in xe_pt_scan_64K()
433 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_scan_64K()
436 xe_res_next(&curs, addr - xe_walk->va_curs_start); in xe_pt_scan_64K()
448 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
449 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
459 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) in xe_pt_is_pte_ps64K()
462 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); in xe_pt_is_pte_ps64K()
463 xe_walk->addr_64K = addr; in xe_pt_is_pte_ps64K()
465 return xe_walk->found_64K; in xe_pt_is_pte_ps64K()
469 xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, in xe_pt_stage_bind_entry() argument
477 u16 pat_index = xe_walk->vma->pat_index; in xe_pt_stage_bind_entry()
478 struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); in xe_pt_stage_bind_entry()
479 struct xe_vm *vm = xe_walk->vm; in xe_pt_stage_bind_entry()
487 struct xe_res_cursor *curs = xe_walk->curs; in xe_pt_stage_bind_entry()
488 bool is_null = xe_vma_is_null(xe_walk->vma); in xe_pt_stage_bind_entry()
490 XE_WARN_ON(xe_walk->va_curs_start != addr); in xe_pt_stage_bind_entry()
492 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : in xe_pt_stage_bind_entry()
493 xe_res_dma(curs) + xe_walk->dma_offset, in xe_pt_stage_bind_entry()
494 xe_walk->vma, pat_index, level); in xe_pt_stage_bind_entry()
495 pte |= xe_walk->default_pte; in xe_pt_stage_bind_entry()
501 if (level == 0 && !xe_parent->is_compact) { in xe_pt_stage_bind_entry()
503 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K; in xe_pt_stage_bind_entry()
505 } else if (XE_WARN_ON(xe_walk->needs_64K)) { in xe_pt_stage_bind_entry()
506 return -EINVAL; in xe_pt_stage_bind_entry()
515 xe_res_next(curs, next - addr); in xe_pt_stage_bind_entry()
516 xe_walk->va_curs_start = next; in xe_pt_stage_bind_entry()
517 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); in xe_pt_stage_bind_entry()
525 * new page table or -directory, which we do if there is no in xe_pt_stage_bind_entry()
529 walk->shifts = xe_normal_pt_shifts; in xe_pt_stage_bind_entry()
530 xe_walk->l0_end_addr = next; in xe_pt_stage_bind_entry()
533 covers = xe_pt_covers(addr, next, level, &xe_walk->base); in xe_pt_stage_bind_entry()
537 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); in xe_pt_stage_bind_entry()
542 round_down(addr, 1ull << walk->shifts[level])); in xe_pt_stage_bind_entry()
545 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); in xe_pt_stage_bind_entry()
547 *child = &xe_child->base; in xe_pt_stage_bind_entry()
556 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && in xe_pt_stage_bind_entry()
558 walk->shifts = xe_compact_pt_shifts; in xe_pt_stage_bind_entry()
559 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; in xe_pt_stage_bind_entry()
561 xe_child->is_compact = true; in xe_pt_stage_bind_entry()
564 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; in xe_pt_stage_bind_entry()
578 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
586 * This function builds a disconnected page-table tree for a given address
590 * shared page-tables, and non-existing shared and non-shared page-tables
616 .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem, in xe_pt_stage_bind()
618 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_bind()
621 if (vma && (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) && in xe_pt_stage_bind()
627 xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); in xe_pt_stage_bind()
637 xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, in xe_pt_stage_bind()
640 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), in xe_pt_stage_bind()
649 ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_bind()
657 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
659 * @addr: The start address within the non-shared pagetable.
660 * @end: The end address within the non-shared pagetable.
661 * @level: The level of the non-shared pagetable.
664 * @offset: Ignored on input, First non-shared entry on output.
665 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
667 * A non-shared page-table has some entries that belong to the address range
676 * Return: true if there were non-shared entries, false otherwise.
683 u64 size = 1ull << walk->shifts[level]; in xe_pt_nonshared_offsets()
703 (*end_offset)--; in xe_pt_nonshared_offsets()
710 /** @base: The walk base-class */
722 static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, in xe_pt_zap_ptes_entry() argument
734 XE_WARN_ON(!level && xe_child->is_compact); in xe_pt_zap_ptes_entry()
738 * with the child of that entry rather than the parent, so need to in xe_pt_zap_ptes_entry()
741 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, in xe_pt_zap_ptes_entry()
743 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, in xe_pt_zap_ptes_entry()
745 (end_offset - offset) * sizeof(u64)); in xe_pt_zap_ptes_entry()
746 xe_walk->needs_invalidate = true; in xe_pt_zap_ptes_entry()
757 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
764 * page-table entrieaso it can either clear the leaf PTEs or
765 * clear the pointers to lower-level page-tables. The caller is required
766 * to hold the necessary locks to ensure neither the page-table connectivity
767 * nor the page-table entries of the range is updated from under us.
782 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_zap_ptes()
784 if (!(vma->tile_present & BIT(tile->id))) in xe_pt_zap_ptes()
787 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_zap_ptes()
799 struct xe_pt_entry *ptes = update->pt_entries; in xe_vm_populate_pgtable()
823 xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL); in xe_pt_abort_bind()
832 lockdep_assert_held(&vm->lock); in xe_pt_commit_locks_assert()
835 lockdep_assert_held_read(&vm->userptr.notifier_lock); in xe_pt_commit_locks_assert()
837 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); in xe_pt_commit_locks_assert()
856 pt->num_live += entries[i].qwords; in xe_pt_commit_bind()
858 if (!pt->level) { in xe_pt_commit_bind()
870 xe_vma_vm(vma)->flags, deferred); in xe_pt_commit_bind()
872 pt_dir->children[j_] = &newpte->base; in xe_pt_commit_bind()
902 vm_dbg(&xe->drm, "%u entries to update\n", num_entries); in xe_vm_dbg_print_entries()
905 struct xe_pt *xe_pt = entry->pt; in xe_vm_dbg_print_entries()
906 u64 page_size = 1ull << xe_pt_shift(xe_pt->level); in xe_vm_dbg_print_entries()
910 xe_assert(xe, !entry->pt->is_compact); in xe_vm_dbg_print_entries()
911 start = entry->ofs * page_size; in xe_vm_dbg_print_entries()
912 end = start + page_size * entry->qwords; in xe_vm_dbg_print_entries()
913 vm_dbg(&xe->drm, in xe_vm_dbg_print_entries()
915 i, xe_pt->level, entry->ofs, entry->qwords, in xe_vm_dbg_print_entries()
927 u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; in xe_pt_userptr_inject_eagain()
930 if (count++ % divisor == divisor - 1) { in xe_pt_userptr_inject_eagain()
931 struct xe_vm *vm = xe_vma_vm(&uvma->vma); in xe_pt_userptr_inject_eagain()
933 uvma->userptr.divisor = divisor << 1; in xe_pt_userptr_inject_eagain()
934 spin_lock(&vm->userptr.invalidated_lock); in xe_pt_userptr_inject_eagain()
935 list_move_tail(&uvma->userptr.invalidate_link, in xe_pt_userptr_inject_eagain()
936 &vm->userptr.invalidated); in xe_pt_userptr_inject_eagain()
937 spin_unlock(&vm->userptr.invalidated_lock); in xe_pt_userptr_inject_eagain()
954 * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks
957 * makes the pre-commit callback error with -EAGAIN if it detects a
959 * @locked: Whether the pre-commit callback locked the userptr notifier lock
965 bool locked; member
969 * This function adds the needed dependencies to a page-table update job
971 * to the same page-table range, wreaking havoc. Initially use a single
984 fence = rtfence->fence; in xe_pt_vm_dependencies()
992 return -ETIME; in xe_pt_vm_dependencies()
995 err = drm_sched_job_add_dependency(&job->drm, fence); in xe_pt_vm_dependencies()
1009 &xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id]; in xe_pt_pre_commit()
1011 return xe_pt_vm_dependencies(pt_update->job, rftree, in xe_pt_pre_commit()
1012 pt_update->start, pt_update->last); in xe_pt_pre_commit()
1019 struct xe_userptr_vma *uvma = to_userptr_vma(pt_update->vma); in xe_pt_userptr_pre_commit()
1020 unsigned long notifier_seq = uvma->userptr.notifier_seq; in xe_pt_userptr_pre_commit()
1021 struct xe_vm *vm = xe_vma_vm(&uvma->vma); in xe_pt_userptr_pre_commit()
1022 int err = xe_pt_vm_dependencies(pt_update->job, in xe_pt_userptr_pre_commit()
1023 &vm->rftree[pt_update->tile_id], in xe_pt_userptr_pre_commit()
1024 pt_update->start, in xe_pt_userptr_pre_commit()
1025 pt_update->last); in xe_pt_userptr_pre_commit()
1030 userptr_update->locked = false; in xe_pt_userptr_pre_commit()
1037 * Note that we don't update the vma->userptr.notifier_seq since in xe_pt_userptr_pre_commit()
1041 down_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1042 if (!mmu_interval_read_retry(&uvma->userptr.notifier, in xe_pt_userptr_pre_commit()
1046 up_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1048 if (userptr_update->bind) in xe_pt_userptr_pre_commit()
1049 return -EAGAIN; in xe_pt_userptr_pre_commit()
1051 notifier_seq = mmu_interval_read_begin(&uvma->userptr.notifier); in xe_pt_userptr_pre_commit()
1055 if (userptr_update->bind && xe_pt_userptr_inject_eagain(uvma)) { in xe_pt_userptr_pre_commit()
1056 up_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1057 return -EAGAIN; in xe_pt_userptr_pre_commit()
1060 userptr_update->locked = true; in xe_pt_userptr_pre_commit()
1107 trace_xe_gt_tlb_invalidation_fence_cb(&ifence->base); in invalidation_fence_cb()
1108 if (!ifence->fence->error) { in invalidation_fence_cb()
1109 queue_work(system_wq, &ifence->work); in invalidation_fence_cb()
1111 ifence->base.base.error = ifence->fence->error; in invalidation_fence_cb()
1112 dma_fence_signal(&ifence->base.base); in invalidation_fence_cb()
1113 dma_fence_put(&ifence->base.base); in invalidation_fence_cb()
1115 dma_fence_put(ifence->fence); in invalidation_fence_cb()
1123 trace_xe_gt_tlb_invalidation_fence_work_func(&ifence->base); in invalidation_fence_work_func()
1124 xe_gt_tlb_invalidation_vma(ifence->gt, &ifence->base, ifence->vma); in invalidation_fence_work_func()
1134 trace_xe_gt_tlb_invalidation_fence_create(&ifence->base); in invalidation_fence_init()
1136 spin_lock_irq(>->tlb_invalidation.lock); in invalidation_fence_init()
1137 dma_fence_init(&ifence->base.base, &invalidation_fence_ops, in invalidation_fence_init()
1138 >->tlb_invalidation.lock, in invalidation_fence_init()
1139 gt->tlb_invalidation.fence_context, in invalidation_fence_init()
1140 ++gt->tlb_invalidation.fence_seqno); in invalidation_fence_init()
1141 spin_unlock_irq(>->tlb_invalidation.lock); in invalidation_fence_init()
1143 INIT_LIST_HEAD(&ifence->base.link); in invalidation_fence_init()
1145 dma_fence_get(&ifence->base.base); /* Ref for caller */ in invalidation_fence_init()
1146 ifence->fence = fence; in invalidation_fence_init()
1147 ifence->gt = gt; in invalidation_fence_init()
1148 ifence->vma = vma; in invalidation_fence_init()
1150 INIT_WORK(&ifence->work, invalidation_fence_work_func); in invalidation_fence_init()
1151 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); in invalidation_fence_init()
1152 if (ret == -ENOENT) { in invalidation_fence_init()
1153 dma_fence_put(ifence->fence); /* Usually dropped in CB */ in invalidation_fence_init()
1154 invalidation_fence_work_func(&ifence->work); in invalidation_fence_init()
1156 dma_fence_put(&ifence->base.base); /* Caller ref */ in invalidation_fence_init()
1157 dma_fence_put(&ifence->base.base); /* Creation ref */ in invalidation_fence_init()
1160 xe_gt_assert(gt, !ret || ret == -ENOENT); in invalidation_fence_init()
1162 return ret && ret != -ENOENT ? ret : 0; in invalidation_fence_init()
1175 if (entry->pt->level > level) in xe_pt_calc_rfence_interval()
1176 level = entry->pt->level; in xe_pt_calc_rfence_interval()
1179 /* Greedy (non-optimal) calculation but simple */ in xe_pt_calc_rfence_interval()
1180 update->base.start = ALIGN_DOWN(xe_vma_start(vma), in xe_pt_calc_rfence_interval()
1182 update->base.last = ALIGN(xe_vma_end(vma), in xe_pt_calc_rfence_interval()
1183 0x1ull << xe_pt_shift(level)) - 1; in xe_pt_calc_rfence_interval()
1187 * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma
1191 * @q: The exec_queue with which to do pipelined page-table updates.
1195 * an unbind in-between.
1197 * This function builds a page-table tree (see xe_pt_stage_bind() for more
1198 * information on page-table building), and the xe_vm_pgtable_update entries
1202 * gpu page-table to the vm main tree, (which can be done either by the
1205 * Return: A valid dma-fence representing the pipelined attachment operation
1218 .tile_id = tile->id, in __xe_pt_bind_vma()
1229 bind_pt_update.locked = false; in __xe_pt_bind_vma()
1233 vm_dbg(&xe_vma_vm(vma)->xe->drm, in __xe_pt_bind_vma()
1249 * automatically when the context is re-enabled by the rebind worker, in __xe_pt_bind_vma()
1255 * non-faulting LR, in particular on user-space batch buffer chaining, in __xe_pt_bind_vma()
1258 if ((rebind && !xe_vm_in_lr_mode(vm) && !vm->batch_invalidate_tlb) || in __xe_pt_bind_vma()
1262 return ERR_PTR(-ENOMEM); in __xe_pt_bind_vma()
1268 return ERR_PTR(-ENOMEM); in __xe_pt_bind_vma()
1271 fence = xe_migrate_update_pgtables(tile->migrate, in __xe_pt_bind_vma()
1277 bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND; in __xe_pt_bind_vma()
1281 err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, in __xe_pt_bind_vma()
1290 int err = invalidation_fence_init(tile->primary_gt, ifence, fence, in __xe_pt_bind_vma()
1297 fence = &ifence->base.base; in __xe_pt_bind_vma()
1306 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) in __xe_pt_bind_vma()
1307 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in __xe_pt_bind_vma()
1310 bind_pt_update.locked ? &deferred : NULL); in __xe_pt_bind_vma()
1313 vma->tile_present |= BIT(tile->id); in __xe_pt_bind_vma()
1315 if (bind_pt_update.locked) { in __xe_pt_bind_vma()
1316 to_userptr_vma(vma)->userptr.initial_bind = true; in __xe_pt_bind_vma()
1317 up_read(&vm->userptr.notifier_lock); in __xe_pt_bind_vma()
1326 if (bind_pt_update.locked) in __xe_pt_bind_vma()
1327 up_read(&vm->userptr.notifier_lock); in __xe_pt_bind_vma()
1338 /** @base: The pagewalk base-class. */
1355 /* @wupd: Structure to track the page-table updates we're building */
1371 unsigned int shift = walk->shifts[level]; in xe_pt_check_kill()
1375 ((next - addr) >> shift) == child->num_live) { in xe_pt_check_kill()
1376 u64 size = 1ull << walk->shifts[level + 1]; in xe_pt_check_kill()
1380 if (xe_walk->modified_start >= addr) in xe_pt_check_kill()
1381 xe_walk->modified_start = round_down(addr, size); in xe_pt_check_kill()
1382 if (xe_walk->modified_end <= next) in xe_pt_check_kill()
1383 xe_walk->modified_end = round_up(next, size); in xe_pt_check_kill()
1391 static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, in xe_pt_stage_unbind_entry() argument
1400 XE_WARN_ON(!level && xe_child->is_compact); in xe_pt_stage_unbind_entry()
1402 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); in xe_pt_stage_unbind_entry()
1408 xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, in xe_pt_stage_unbind_post_descend() argument
1418 u64 size = 1ull << walk->shifts[--level]; in xe_pt_stage_unbind_post_descend()
1421 addr = xe_walk->modified_start; in xe_pt_stage_unbind_post_descend()
1423 next = xe_walk->modified_end; in xe_pt_stage_unbind_post_descend()
1425 /* Parent == *child is the root pt. Don't kill it. */ in xe_pt_stage_unbind_post_descend()
1426 if (parent != *child && in xe_pt_stage_unbind_post_descend()
1434 (void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false); in xe_pt_stage_unbind_post_descend()
1435 xe_walk->wupd.updates[level].update->qwords = end_offset - offset; in xe_pt_stage_unbind_post_descend()
1446 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1450 * @entries: Caller-provided storage for the update structures.
1452 * Builds page-table update structures for an unbind operation. The function
1453 * will attempt to remove all page-tables that we're the only user
1455 * same critical section that blocks racing binds to the same page-table tree.
1473 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_unbind()
1475 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_unbind()
1487 struct xe_vma *vma = pt_update->vma; in xe_migrate_clear_pgtable_callback()
1488 u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level); in xe_migrate_clear_pgtable_callback()
1491 if (map && map->is_iomem) in xe_migrate_clear_pgtable_callback()
1496 memset64(map->vaddr + qword_ofs * sizeof(u64), empty, in xe_migrate_clear_pgtable_callback()
1513 struct xe_pt *pt = entry->pt; in xe_pt_commit_unbind()
1515 pt->num_live -= entry->qwords; in xe_pt_commit_unbind()
1516 if (pt->level) { in xe_pt_commit_unbind()
1520 for (i = entry->ofs; i < entry->ofs + entry->qwords; in xe_pt_commit_unbind()
1524 xe_vma_vm(vma)->flags, deferred); in xe_pt_commit_unbind()
1526 pt_dir->children[i] = NULL; in xe_pt_commit_unbind()
1543 * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma
1547 * @q: The exec_queue with which to do pipelined page-table updates.
1552 * operations needed to detach the page-table tree to be destroyed from the
1555 * pipelined detachment of the gpu page-table from the vm main tree,
1557 * detached page-table tree.
1559 * Return: A valid dma-fence representing the pipelined detachment operation
1572 .tile_id = tile->id, in __xe_pt_unbind_vma()
1586 vm_dbg(&xe_vma_vm(vma)->xe->drm, in __xe_pt_unbind_vma()
1599 return ERR_PTR(-ENOMEM); in __xe_pt_unbind_vma()
1604 return ERR_PTR(-ENOMEM); in __xe_pt_unbind_vma()
1612 fence = xe_migrate_update_pgtables(tile->migrate, in __xe_pt_unbind_vma()
1614 vm->q[tile->id], in __xe_pt_unbind_vma()
1621 err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, in __xe_pt_unbind_vma()
1629 err = invalidation_fence_init(tile->primary_gt, ifence, fence, vma); in __xe_pt_unbind_vma()
1635 fence = &ifence->base.base; in __xe_pt_unbind_vma()
1642 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) in __xe_pt_unbind_vma()
1643 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in __xe_pt_unbind_vma()
1646 unbind_pt_update.locked ? &deferred : NULL); in __xe_pt_unbind_vma()
1647 vma->tile_present &= ~BIT(tile->id); in __xe_pt_unbind_vma()
1653 if (!vma->tile_present) in __xe_pt_unbind_vma()
1654 list_del_init(&vma->combined_links.rebind); in __xe_pt_unbind_vma()
1656 if (unbind_pt_update.locked) { in __xe_pt_unbind_vma()
1659 if (!vma->tile_present) { in __xe_pt_unbind_vma()
1660 spin_lock(&vm->userptr.invalidated_lock); in __xe_pt_unbind_vma()
1661 list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); in __xe_pt_unbind_vma()
1662 spin_unlock(&vm->userptr.invalidated_lock); in __xe_pt_unbind_vma()
1664 up_read(&vm->userptr.notifier_lock); in __xe_pt_unbind_vma()