Lines Matching refs:gpusvm
282 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start, in drm_gpusvm_notifier_find() argument
287 itree = interval_tree_iter_first(&gpusvm->root, start, end - 1); in drm_gpusvm_notifier_find()
338 struct drm_gpusvm *gpusvm = notifier->gpusvm; in drm_gpusvm_notifier_invalidate() local
343 down_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
345 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range); in drm_gpusvm_notifier_invalidate()
346 up_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
383 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, in drm_gpusvm_init() argument
401 gpusvm->name = name; in drm_gpusvm_init()
402 gpusvm->drm = drm; in drm_gpusvm_init()
403 gpusvm->mm = mm; in drm_gpusvm_init()
404 gpusvm->mm_start = mm_start; in drm_gpusvm_init()
405 gpusvm->mm_range = mm_range; in drm_gpusvm_init()
406 gpusvm->notifier_size = notifier_size; in drm_gpusvm_init()
407 gpusvm->ops = ops; in drm_gpusvm_init()
408 gpusvm->chunk_sizes = chunk_sizes; in drm_gpusvm_init()
409 gpusvm->num_chunks = num_chunks; in drm_gpusvm_init()
411 gpusvm->root = RB_ROOT_CACHED; in drm_gpusvm_init()
412 INIT_LIST_HEAD(&gpusvm->notifier_list); in drm_gpusvm_init()
414 init_rwsem(&gpusvm->notifier_lock); in drm_gpusvm_init()
417 might_lock(&gpusvm->notifier_lock); in drm_gpusvm_init()
421 gpusvm->lock_dep_map = NULL; in drm_gpusvm_init()
446 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_insert() argument
452 interval_tree_insert(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_insert()
458 head = &gpusvm->notifier_list; in drm_gpusvm_notifier_insert()
470 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_remove() argument
473 interval_tree_remove(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_remove()
484 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm) in drm_gpusvm_fini() argument
488 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) { in drm_gpusvm_fini()
499 drm_gpusvm_range_remove(gpusvm, range); in drm_gpusvm_fini()
502 if (gpusvm->mm) in drm_gpusvm_fini()
503 mmdrop(gpusvm->mm); in drm_gpusvm_fini()
504 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root)); in drm_gpusvm_fini()
518 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr) in drm_gpusvm_notifier_alloc() argument
522 if (gpusvm->ops->notifier_alloc) in drm_gpusvm_notifier_alloc()
523 notifier = gpusvm->ops->notifier_alloc(); in drm_gpusvm_notifier_alloc()
530 notifier->gpusvm = gpusvm; in drm_gpusvm_notifier_alloc()
531 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size); in drm_gpusvm_notifier_alloc()
532 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1; in drm_gpusvm_notifier_alloc()
547 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_free() argument
552 if (gpusvm->ops->notifier_free) in drm_gpusvm_notifier_free()
553 gpusvm->ops->notifier_free(notifier); in drm_gpusvm_notifier_free()
582 drm_gpusvm_notifier_lock(notifier->gpusvm); in drm_gpusvm_range_insert()
592 drm_gpusvm_notifier_unlock(notifier->gpusvm); in drm_gpusvm_range_insert()
622 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_alloc() argument
629 if (gpusvm->ops->range_alloc) in drm_gpusvm_range_alloc()
630 range = gpusvm->ops->range_alloc(gpusvm); in drm_gpusvm_range_alloc()
638 range->gpusvm = gpusvm; in drm_gpusvm_range_alloc()
691 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_check_pages() argument
709 mmap_assert_locked(gpusvm->mm); in drm_gpusvm_check_pages()
792 mmap_read_lock(range->gpusvm->mm); in drm_gpusvm_scan_mm()
806 mmap_read_unlock(range->gpusvm->mm); in drm_gpusvm_scan_mm()
810 drm_gpusvm_notifier_lock(range->gpusvm); in drm_gpusvm_scan_mm()
812 drm_gpusvm_notifier_unlock(range->gpusvm); in drm_gpusvm_scan_mm()
860 drm_gpusvm_notifier_unlock(range->gpusvm); in drm_gpusvm_scan_mm()
885 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_chunk_size() argument
898 for (; i < gpusvm->num_chunks; ++i) { in drm_gpusvm_range_chunk_size()
899 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
900 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
909 if (i == gpusvm->num_chunks) in drm_gpusvm_range_chunk_size()
939 !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) { in drm_gpusvm_range_chunk_size()
955 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) in drm_gpusvm_driver_lock_held() argument
957 if ((gpusvm)->lock_dep_map) in drm_gpusvm_driver_lock_held()
958 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0)); in drm_gpusvm_driver_lock_held()
961 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) in drm_gpusvm_driver_lock_held() argument
976 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, in drm_gpusvm_find_vma_start() argument
980 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_find_vma_start()
1014 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_find_or_insert() argument
1022 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_find_or_insert()
1029 drm_gpusvm_driver_lock_held(gpusvm); in drm_gpusvm_range_find_or_insert()
1031 if (fault_addr < gpusvm->mm_start || in drm_gpusvm_range_find_or_insert()
1032 fault_addr > gpusvm->mm_start + gpusvm->mm_range) in drm_gpusvm_range_find_or_insert()
1038 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1); in drm_gpusvm_range_find_or_insert()
1040 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr); in drm_gpusvm_range_find_or_insert()
1079 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas, in drm_gpusvm_range_find_or_insert()
1089 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size, in drm_gpusvm_range_find_or_insert()
1098 drm_gpusvm_notifier_insert(gpusvm, notifier); in drm_gpusvm_range_find_or_insert()
1112 drm_gpusvm_notifier_free(gpusvm, notifier); in drm_gpusvm_range_find_or_insert()
1128 static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, in __drm_gpusvm_unmap_pages() argument
1133 struct device *dev = gpusvm->drm->dev; in __drm_gpusvm_unmap_pages()
1136 lockdep_assert_held(&gpusvm->notifier_lock); in __drm_gpusvm_unmap_pages()
1174 static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, in __drm_gpusvm_free_pages() argument
1177 lockdep_assert_held(&gpusvm->notifier_lock); in __drm_gpusvm_free_pages()
1195 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_free_pages() argument
1199 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_free_pages()
1200 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); in drm_gpusvm_free_pages()
1201 __drm_gpusvm_free_pages(gpusvm, svm_pages); in drm_gpusvm_free_pages()
1202 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_free_pages()
1215 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_remove() argument
1222 drm_gpusvm_driver_lock_held(gpusvm); in drm_gpusvm_range_remove()
1224 notifier = drm_gpusvm_notifier_find(gpusvm, in drm_gpusvm_range_remove()
1230 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_range_remove()
1231 __drm_gpusvm_unmap_pages(gpusvm, &range->pages, npages); in drm_gpusvm_range_remove()
1232 __drm_gpusvm_free_pages(gpusvm, &range->pages); in drm_gpusvm_range_remove()
1234 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_remove()
1241 drm_gpusvm_notifier_remove(gpusvm, notifier); in drm_gpusvm_range_remove()
1242 drm_gpusvm_notifier_free(gpusvm, notifier); in drm_gpusvm_range_remove()
1276 struct drm_gpusvm *gpusvm = range->gpusvm; in drm_gpusvm_range_destroy() local
1278 if (gpusvm->ops->range_free) in drm_gpusvm_range_destroy()
1279 gpusvm->ops->range_free(range); in drm_gpusvm_range_destroy()
1311 static bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm, in drm_gpusvm_pages_valid() argument
1314 lockdep_assert_held(&gpusvm->notifier_lock); in drm_gpusvm_pages_valid()
1333 bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_pages_valid() argument
1336 return drm_gpusvm_pages_valid(gpusvm, &range->pages); in drm_gpusvm_range_pages_valid()
1350 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, in drm_gpusvm_pages_valid_unlocked() argument
1358 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_pages_valid_unlocked()
1359 pages_valid = drm_gpusvm_pages_valid(gpusvm, svm_pages); in drm_gpusvm_pages_valid_unlocked()
1361 __drm_gpusvm_free_pages(gpusvm, svm_pages); in drm_gpusvm_pages_valid_unlocked()
1362 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_pages_valid_unlocked()
1382 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_get_pages() argument
1417 if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) in drm_gpusvm_get_pages()
1455 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_get_pages()
1459 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_get_pages()
1465 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_get_pages()
1472 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_get_pages()
1505 if (drm_WARN_ON(gpusvm->drm, !dpagemap)) { in drm_gpusvm_get_pages()
1517 gpusvm->drm->dev, in drm_gpusvm_get_pages()
1520 if (dma_mapping_error(gpusvm->drm->dev, in drm_gpusvm_get_pages()
1539 addr = dma_map_page(gpusvm->drm->dev, in drm_gpusvm_get_pages()
1543 if (dma_mapping_error(gpusvm->drm->dev, addr)) { in drm_gpusvm_get_pages()
1567 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_get_pages()
1575 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); in drm_gpusvm_get_pages()
1576 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_get_pages()
1596 int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_get_pages() argument
1600 return drm_gpusvm_get_pages(gpusvm, &range->pages, gpusvm->mm, in drm_gpusvm_range_get_pages()
1620 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_unmap_pages() argument
1626 lockdep_assert_held_write(&gpusvm->notifier_lock); in drm_gpusvm_unmap_pages()
1628 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_unmap_pages()
1630 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); in drm_gpusvm_unmap_pages()
1633 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_unmap_pages()
1649 void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_unmap_pages() argument
1656 return drm_gpusvm_unmap_pages(gpusvm, &range->pages, npages, ctx); in drm_gpusvm_range_unmap_pages()
1669 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_evict() argument
1686 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_evict()
1725 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, in drm_gpusvm_has_mapping() argument
1730 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) { in drm_gpusvm_has_mapping()
1752 lockdep_assert_held_write(&range->gpusvm->notifier_lock); in drm_gpusvm_range_set_unmapped()