Lines Matching +full:long +full:- +full:ram +full:- +full:code
1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
9 #include <linux/dma-mapping.h>
28 * for GPU-accelerated applications by allowing memory sharing and
33 * - Notifiers:
36 * recommendation of 512M or larger. They maintain a Red-BlacK tree and a
38 * tracked within a GPU SVM Red-BlacK tree and list and are dynamically
41 * - Ranges:
48 * event. As mentioned above, ranges are tracked in a notifier's Red-Black
51 * - Operations:
52 * Define the interface for driver-specific GPU SVM operations such as
55 * - Device Memory Allocations:
59 * - Device Memory Operations:
60 * Define the interface for driver-specific device memory operations
66 * driver-specific virtual functions (vfuncs). This infrastructure is sufficient
72 * - GPU page fault handler:
76 * - Garbage collector:
81 * - Notifier callback:
96 * This lock corresponds to the ``driver->update`` lock mentioned in
98 * global lock to a per-notifier lock if finer-grained locking is deemed
104 * denoted as 'driver_svm_lock' in code examples. Finer grained driver side
113 * The migration support is quite simple, allowing migration between RAM and
115 * not support mixing RAM and device memory pages within a range. This means
117 * upon CPU fault, the entire range is migrated to RAM. Mixed RAM and device
121 * simplifies the implementation, and range sizes are driver-defined and should
134 * complicated. Given that partial unmappings are rare and driver-defined range
140 * remaining pages back to RAM.
150 * The generic code provided does not include logic for complex migration
152 * potentially required driver locking (e.g., DMA-resv locks).
156 * .. code-block:: c
168 * err = -EAGAIN;
174 * int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr,
175 * unsigned long gpuva_start, unsigned long gpuva_end)
205 * if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { // CPU mappings changed
206 * if (err == -EOPNOTSUPP)
214 * if (err == -EAGAIN) // CPU mappings changed
224 * .. code-block:: c
231 * // Partial unmap, migrate any remaining device memory pages back to RAM
232 * if (range->flags.partial_unmap)
249 * .. code-block:: c
258 * driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
260 * drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
261 * mmu_range->end) {
264 * if (mmu_range->event != MMU_NOTIFY_UNMAP)
274 * npages_in_range() - Calculate the number of pages in a given range
285 static unsigned long
286 npages_in_range(unsigned long start, unsigned long end) in npages_in_range()
288 return (end - start) >> PAGE_SHIFT; in npages_in_range()
292 * struct drm_gpusvm_zdd - GPU SVM zone device data
299 * page->zone_device_data. It provides infrastructure for looking up a device
312 * drm_gpusvm_zdd_alloc() - Allocate a zdd structure.
329 kref_init(&zdd->refcount); in drm_gpusvm_zdd_alloc()
330 zdd->devmem_allocation = NULL; in drm_gpusvm_zdd_alloc()
331 zdd->device_private_page_owner = device_private_page_owner; in drm_gpusvm_zdd_alloc()
337 * drm_gpusvm_zdd_get() - Get a reference to a zdd structure.
346 kref_get(&zdd->refcount); in drm_gpusvm_zdd_get()
351 * drm_gpusvm_zdd_destroy() - Destroy a zdd structure.
360 struct drm_gpusvm_devmem *devmem = zdd->devmem_allocation; in drm_gpusvm_zdd_destroy()
363 complete_all(&devmem->detached); in drm_gpusvm_zdd_destroy()
364 if (devmem->ops->devmem_release) in drm_gpusvm_zdd_destroy()
365 devmem->ops->devmem_release(devmem); in drm_gpusvm_zdd_destroy()
371 * drm_gpusvm_zdd_put() - Put a zdd reference.
379 kref_put(&zdd->refcount, drm_gpusvm_zdd_destroy); in drm_gpusvm_zdd_put()
383 * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
391 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, in drm_gpusvm_range_find()
392 unsigned long end) in drm_gpusvm_range_find()
396 itree = interval_tree_iter_first(¬ifier->root, start, end - 1); in drm_gpusvm_range_find()
406 * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier
423 * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list
433 if (notifier && !list_is_last(¬ifier->entry, in __drm_gpusvm_notifier_next()
434 ¬ifier->gpusvm->notifier_list)) in __drm_gpusvm_notifier_next()
441 notifier_iter_first(struct rb_root_cached *root, unsigned long start, in notifier_iter_first()
442 unsigned long last) in notifier_iter_first()
455 * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm
464 for ((notifier__) = notifier_iter_first(&(gpusvm__)->root, (start__), (end__) - 1); \
469 * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm
480 for ((notifier__) = notifier_iter_first(&(gpusvm__)->root, (start__), (end__) - 1), \
486 * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
493 * gpusvm->notifier_lock.
500 unsigned long cur_seq) in drm_gpusvm_notifier_invalidate()
504 struct drm_gpusvm *gpusvm = notifier->gpusvm; in drm_gpusvm_notifier_invalidate()
509 down_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
511 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range); in drm_gpusvm_notifier_invalidate()
512 up_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
518 * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
525 * drm_gpusvm_init() - Initialize the GPU SVM.
542 * Return: 0 on success, a negative error code on failure.
547 unsigned long mm_start, unsigned long mm_range, in drm_gpusvm_init()
548 unsigned long notifier_size, in drm_gpusvm_init()
550 const unsigned long *chunk_sizes, int num_chunks) in drm_gpusvm_init()
552 if (!ops->invalidate || !num_chunks) in drm_gpusvm_init()
553 return -EINVAL; in drm_gpusvm_init()
555 gpusvm->name = name; in drm_gpusvm_init()
556 gpusvm->drm = drm; in drm_gpusvm_init()
557 gpusvm->mm = mm; in drm_gpusvm_init()
558 gpusvm->device_private_page_owner = device_private_page_owner; in drm_gpusvm_init()
559 gpusvm->mm_start = mm_start; in drm_gpusvm_init()
560 gpusvm->mm_range = mm_range; in drm_gpusvm_init()
561 gpusvm->notifier_size = notifier_size; in drm_gpusvm_init()
562 gpusvm->ops = ops; in drm_gpusvm_init()
563 gpusvm->chunk_sizes = chunk_sizes; in drm_gpusvm_init()
564 gpusvm->num_chunks = num_chunks; in drm_gpusvm_init()
567 gpusvm->root = RB_ROOT_CACHED; in drm_gpusvm_init()
568 INIT_LIST_HEAD(&gpusvm->notifier_list); in drm_gpusvm_init()
570 init_rwsem(&gpusvm->notifier_lock); in drm_gpusvm_init()
573 might_lock(&gpusvm->notifier_lock); in drm_gpusvm_init()
577 gpusvm->lock_dep_map = NULL; in drm_gpusvm_init()
585 * drm_gpusvm_notifier_find() - Find GPU SVM notifier
595 unsigned long fault_addr) in drm_gpusvm_notifier_find()
597 return notifier_iter_first(&gpusvm->root, fault_addr, fault_addr + 1); in drm_gpusvm_notifier_find()
601 * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node
612 * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
624 interval_tree_insert(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_insert()
626 node = rb_prev(¬ifier->itree.rb); in drm_gpusvm_notifier_insert()
628 head = &(to_drm_gpusvm_notifier(node))->entry; in drm_gpusvm_notifier_insert()
630 head = &gpusvm->notifier_list; in drm_gpusvm_notifier_insert()
632 list_add(¬ifier->entry, head); in drm_gpusvm_notifier_insert()
636 * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
645 interval_tree_remove(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_remove()
646 list_del(¬ifier->entry); in drm_gpusvm_notifier_remove()
650 * drm_gpusvm_fini() - Finalize the GPU SVM.
666 mmu_interval_notifier_remove(¬ifier->notifier); in drm_gpusvm_fini()
667 notifier->flags.removed = true; in drm_gpusvm_fini()
674 mmdrop(gpusvm->mm); in drm_gpusvm_fini()
675 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root)); in drm_gpusvm_fini()
680 * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
689 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr) in drm_gpusvm_notifier_alloc()
693 if (gpusvm->ops->notifier_alloc) in drm_gpusvm_notifier_alloc()
694 notifier = gpusvm->ops->notifier_alloc(); in drm_gpusvm_notifier_alloc()
699 return ERR_PTR(-ENOMEM); in drm_gpusvm_notifier_alloc()
701 notifier->gpusvm = gpusvm; in drm_gpusvm_notifier_alloc()
702 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size); in drm_gpusvm_notifier_alloc()
703 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1; in drm_gpusvm_notifier_alloc()
704 INIT_LIST_HEAD(¬ifier->entry); in drm_gpusvm_notifier_alloc()
705 notifier->root = RB_ROOT_CACHED; in drm_gpusvm_notifier_alloc()
706 INIT_LIST_HEAD(¬ifier->range_list); in drm_gpusvm_notifier_alloc()
712 * drm_gpusvm_notifier_free() - Free GPU SVM notifier
721 WARN_ON(!RB_EMPTY_ROOT(¬ifier->root.rb_root)); in drm_gpusvm_notifier_free()
723 if (gpusvm->ops->notifier_free) in drm_gpusvm_notifier_free()
724 gpusvm->ops->notifier_free(notifier); in drm_gpusvm_notifier_free()
730 * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node
741 * drm_gpusvm_range_insert() - Insert GPU SVM range
753 drm_gpusvm_notifier_lock(notifier->gpusvm); in drm_gpusvm_range_insert()
754 interval_tree_insert(&range->itree, ¬ifier->root); in drm_gpusvm_range_insert()
756 node = rb_prev(&range->itree.rb); in drm_gpusvm_range_insert()
758 head = &(to_drm_gpusvm_range(node))->entry; in drm_gpusvm_range_insert()
760 head = ¬ifier->range_list; in drm_gpusvm_range_insert()
762 list_add(&range->entry, head); in drm_gpusvm_range_insert()
763 drm_gpusvm_notifier_unlock(notifier->gpusvm); in drm_gpusvm_range_insert()
767 * __drm_gpusvm_range_remove() - Remove GPU SVM range
776 interval_tree_remove(&range->itree, ¬ifier->root); in __drm_gpusvm_range_remove()
777 list_del(&range->entry); in __drm_gpusvm_range_remove()
781 * drm_gpusvm_range_alloc() - Allocate GPU SVM range
795 unsigned long fault_addr, unsigned long chunk_size, in drm_gpusvm_range_alloc()
800 if (gpusvm->ops->range_alloc) in drm_gpusvm_range_alloc()
801 range = gpusvm->ops->range_alloc(gpusvm); in drm_gpusvm_range_alloc()
806 return ERR_PTR(-ENOMEM); in drm_gpusvm_range_alloc()
808 kref_init(&range->refcount); in drm_gpusvm_range_alloc()
809 range->gpusvm = gpusvm; in drm_gpusvm_range_alloc()
810 range->notifier = notifier; in drm_gpusvm_range_alloc()
811 range->itree.start = ALIGN_DOWN(fault_addr, chunk_size); in drm_gpusvm_range_alloc()
812 range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1; in drm_gpusvm_range_alloc()
813 INIT_LIST_HEAD(&range->entry); in drm_gpusvm_range_alloc()
814 range->notifier_seq = LONG_MAX; in drm_gpusvm_range_alloc()
815 range->flags.migrate_devmem = migrate_devmem ? 1 : 0; in drm_gpusvm_range_alloc()
821 * drm_gpusvm_check_pages() - Check pages
834 unsigned long start, unsigned long end) in drm_gpusvm_check_pages()
838 .notifier = ¬ifier->notifier, in drm_gpusvm_check_pages()
841 .dev_private_owner = gpusvm->device_private_page_owner, in drm_gpusvm_check_pages()
843 unsigned long timeout = in drm_gpusvm_check_pages()
845 unsigned long *pfns; in drm_gpusvm_check_pages()
846 unsigned long npages = npages_in_range(start, end); in drm_gpusvm_check_pages()
849 mmap_assert_locked(gpusvm->mm); in drm_gpusvm_check_pages()
855 hmm_range.notifier_seq = mmu_interval_read_begin(¬ifier->notifier); in drm_gpusvm_check_pages()
860 if (err == -EBUSY) { in drm_gpusvm_check_pages()
865 mmu_interval_read_begin(¬ifier->notifier); in drm_gpusvm_check_pages()
875 err = -EFAULT; in drm_gpusvm_check_pages()
887 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range
902 static unsigned long
906 unsigned long fault_addr, in drm_gpusvm_range_chunk_size()
907 unsigned long gpuva_start, in drm_gpusvm_range_chunk_size()
908 unsigned long gpuva_end, in drm_gpusvm_range_chunk_size()
909 unsigned long check_pages_threshold) in drm_gpusvm_range_chunk_size()
911 unsigned long start, end; in drm_gpusvm_range_chunk_size()
915 for (; i < gpusvm->num_chunks; ++i) { in drm_gpusvm_range_chunk_size()
916 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
917 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
919 if (start >= vas->vm_start && end <= vas->vm_end && in drm_gpusvm_range_chunk_size()
926 if (i == gpusvm->num_chunks) in drm_gpusvm_range_chunk_size()
933 if (end - start != SZ_4K) { in drm_gpusvm_range_chunk_size()
944 * this check, or prefault, on BMG 'xe_exec_system_allocator --r in drm_gpusvm_range_chunk_size()
945 * process-many-malloc' fails. In the failure case, each process in drm_gpusvm_range_chunk_size()
952 * problem goes away if 'xe_exec_system_allocator --r in drm_gpusvm_range_chunk_size()
953 * process-many-malloc' mallocs at least 64k at a time. in drm_gpusvm_range_chunk_size()
955 if (end - start <= check_pages_threshold && in drm_gpusvm_range_chunk_size()
962 return end - start; in drm_gpusvm_range_chunk_size()
967 * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held
974 if ((gpusvm)->lock_dep_map) in drm_gpusvm_driver_lock_held()
975 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0)); in drm_gpusvm_driver_lock_held()
984 * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range
998 unsigned long fault_addr, in drm_gpusvm_range_find_or_insert()
999 unsigned long gpuva_start, in drm_gpusvm_range_find_or_insert()
1000 unsigned long gpuva_end, in drm_gpusvm_range_find_or_insert()
1005 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_find_or_insert()
1008 unsigned long chunk_size; in drm_gpusvm_range_find_or_insert()
1014 if (fault_addr < gpusvm->mm_start || in drm_gpusvm_range_find_or_insert()
1015 fault_addr > gpusvm->mm_start + gpusvm->mm_range) in drm_gpusvm_range_find_or_insert()
1016 return ERR_PTR(-EINVAL); in drm_gpusvm_range_find_or_insert()
1019 return ERR_PTR(-EFAULT); in drm_gpusvm_range_find_or_insert()
1029 err = mmu_interval_notifier_insert(¬ifier->notifier, in drm_gpusvm_range_find_or_insert()
1042 err = -ENOENT; in drm_gpusvm_range_find_or_insert()
1046 if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) { in drm_gpusvm_range_find_or_insert()
1047 err = -EPERM; in drm_gpusvm_range_find_or_insert()
1055 * XXX: Short-circuiting migration based on migrate_vma_* current in drm_gpusvm_range_find_or_insert()
1059 migrate_devmem = ctx->devmem_possible && in drm_gpusvm_range_find_or_insert()
1065 ctx->check_pages_threshold); in drm_gpusvm_range_find_or_insert()
1067 err = -EINVAL; in drm_gpusvm_range_find_or_insert()
1091 mmu_interval_notifier_remove(¬ifier->notifier); in drm_gpusvm_range_find_or_insert()
1102 * __drm_gpusvm_range_unmap_pages() - Unmap pages associated with a GPU SVM range (internal)
1112 unsigned long npages) in __drm_gpusvm_range_unmap_pages()
1114 unsigned long i, j; in __drm_gpusvm_range_unmap_pages()
1115 struct drm_pagemap *dpagemap = range->dpagemap; in __drm_gpusvm_range_unmap_pages()
1116 struct device *dev = gpusvm->drm->dev; in __drm_gpusvm_range_unmap_pages()
1118 lockdep_assert_held(&gpusvm->notifier_lock); in __drm_gpusvm_range_unmap_pages()
1120 if (range->flags.has_dma_mapping) { in __drm_gpusvm_range_unmap_pages()
1122 .__flags = range->flags.__flags, in __drm_gpusvm_range_unmap_pages()
1126 struct drm_pagemap_device_addr *addr = &range->dma_addr[j]; in __drm_gpusvm_range_unmap_pages()
1128 if (addr->proto == DRM_INTERCONNECT_SYSTEM) in __drm_gpusvm_range_unmap_pages()
1130 addr->addr, in __drm_gpusvm_range_unmap_pages()
1131 PAGE_SIZE << addr->order, in __drm_gpusvm_range_unmap_pages()
1132 addr->dir); in __drm_gpusvm_range_unmap_pages()
1133 else if (dpagemap && dpagemap->ops->device_unmap) in __drm_gpusvm_range_unmap_pages()
1134 dpagemap->ops->device_unmap(dpagemap, in __drm_gpusvm_range_unmap_pages()
1136 i += 1 << addr->order; in __drm_gpusvm_range_unmap_pages()
1142 WRITE_ONCE(range->flags.__flags, flags.__flags); in __drm_gpusvm_range_unmap_pages()
1144 range->dpagemap = NULL; in __drm_gpusvm_range_unmap_pages()
1149 * drm_gpusvm_range_free_pages() - Free pages associated with a GPU SVM range
1158 lockdep_assert_held(&gpusvm->notifier_lock); in drm_gpusvm_range_free_pages()
1160 if (range->dma_addr) { in drm_gpusvm_range_free_pages()
1161 kvfree(range->dma_addr); in drm_gpusvm_range_free_pages()
1162 range->dma_addr = NULL; in drm_gpusvm_range_free_pages()
1167 * drm_gpusvm_range_remove() - Remove GPU SVM range
1178 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), in drm_gpusvm_range_remove()
1197 if (RB_EMPTY_ROOT(¬ifier->root.rb_root)) { in drm_gpusvm_range_remove()
1198 if (!notifier->flags.removed) in drm_gpusvm_range_remove()
1199 mmu_interval_notifier_remove(¬ifier->notifier); in drm_gpusvm_range_remove()
1207 * drm_gpusvm_range_get() - Get a reference to GPU SVM range
1217 kref_get(&range->refcount); in drm_gpusvm_range_get()
1224 * drm_gpusvm_range_destroy() - Destroy GPU SVM range
1228 * reaches zero. If a custom range-free function is provided, it is invoked to
1235 struct drm_gpusvm *gpusvm = range->gpusvm; in drm_gpusvm_range_destroy()
1237 if (gpusvm->ops->range_free) in drm_gpusvm_range_destroy()
1238 gpusvm->ops->range_free(range); in drm_gpusvm_range_destroy()
1244 * drm_gpusvm_range_put() - Put a reference to GPU SVM range
1252 kref_put(&range->refcount, drm_gpusvm_range_destroy); in drm_gpusvm_range_put()
1257 * drm_gpusvm_range_pages_valid() - GPU SVM range pages valid
1262 * called holding gpusvm->notifier_lock and as the last step before committing a
1273 lockdep_assert_held(&gpusvm->notifier_lock); in drm_gpusvm_range_pages_valid()
1275 return range->flags.has_devmem_pages || range->flags.has_dma_mapping; in drm_gpusvm_range_pages_valid()
1280 * drm_gpusvm_range_pages_valid_unlocked() - GPU SVM range pages valid unlocked
1285 * called without holding gpusvm->notifier_lock.
1295 if (!range->dma_addr) in drm_gpusvm_range_pages_valid_unlocked()
1308 * drm_gpusvm_range_get_pages() - Get pages for a GPU SVM range
1316 * Return: 0 on success, negative error code on failure.
1322 struct mmu_interval_notifier *notifier = &range->notifier->notifier; in drm_gpusvm_range_get_pages()
1324 .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 : in drm_gpusvm_range_get_pages()
1329 .dev_private_owner = gpusvm->device_private_page_owner, in drm_gpusvm_range_get_pages()
1331 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_get_pages()
1333 unsigned long timeout = in drm_gpusvm_range_get_pages()
1335 unsigned long i, j; in drm_gpusvm_range_get_pages()
1336 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), in drm_gpusvm_range_get_pages()
1338 unsigned long num_dma_mapped; in drm_gpusvm_range_get_pages()
1340 unsigned long *pfns; in drm_gpusvm_range_get_pages()
1354 return -ENOMEM; in drm_gpusvm_range_get_pages()
1357 err = -EFAULT; in drm_gpusvm_range_get_pages()
1367 if (err == -EBUSY) { in drm_gpusvm_range_get_pages()
1390 flags.__flags = range->flags.__flags; in drm_gpusvm_range_get_pages()
1393 err = -EFAULT; in drm_gpusvm_range_get_pages()
1403 if (!range->dma_addr) { in drm_gpusvm_range_get_pages()
1406 range->dma_addr = kvmalloc_array(npages, in drm_gpusvm_range_get_pages()
1407 sizeof(*range->dma_addr), in drm_gpusvm_range_get_pages()
1409 if (!range->dma_addr) { in drm_gpusvm_range_get_pages()
1410 err = -ENOMEM; in drm_gpusvm_range_get_pages()
1424 if (zdd != page->zone_device_data && i > 0) { in drm_gpusvm_range_get_pages()
1425 err = -EOPNOTSUPP; in drm_gpusvm_range_get_pages()
1428 zdd = page->zone_device_data; in drm_gpusvm_range_get_pages()
1431 err = -EOPNOTSUPP; in drm_gpusvm_range_get_pages()
1436 dpagemap = zdd->devmem_allocation->dpagemap; in drm_gpusvm_range_get_pages()
1437 if (drm_WARN_ON(gpusvm->drm, !dpagemap)) { in drm_gpusvm_range_get_pages()
1443 err = -EAGAIN; in drm_gpusvm_range_get_pages()
1447 range->dma_addr[j] = in drm_gpusvm_range_get_pages()
1448 dpagemap->ops->device_map(dpagemap, in drm_gpusvm_range_get_pages()
1449 gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1452 if (dma_mapping_error(gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1453 range->dma_addr[j].addr)) { in drm_gpusvm_range_get_pages()
1454 err = -EFAULT; in drm_gpusvm_range_get_pages()
1463 err = -EOPNOTSUPP; in drm_gpusvm_range_get_pages()
1467 if (ctx->devmem_only) { in drm_gpusvm_range_get_pages()
1468 err = -EFAULT; in drm_gpusvm_range_get_pages()
1472 addr = dma_map_page(gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1476 if (dma_mapping_error(gpusvm->drm->dev, addr)) { in drm_gpusvm_range_get_pages()
1477 err = -EFAULT; in drm_gpusvm_range_get_pages()
1481 range->dma_addr[j] = drm_pagemap_device_addr_encode in drm_gpusvm_range_get_pages()
1492 range->dpagemap = dpagemap; in drm_gpusvm_range_get_pages()
1496 WRITE_ONCE(range->flags.__flags, flags.__flags); in drm_gpusvm_range_get_pages()
1501 range->notifier_seq = hmm_range.notifier_seq; in drm_gpusvm_range_get_pages()
1510 if (err == -EAGAIN) in drm_gpusvm_range_get_pages()
1517 * drm_gpusvm_range_unmap_pages() - Unmap pages associated with a GPU SVM range
1523 * is set, it is assumed that gpusvm->notifier_lock is held in write mode; if it
1524 * is clear, it acquires gpusvm->notifier_lock in read mode. Must be called on
1525 * each GPU SVM range attached to notifier in gpusvm->ops->invalidate for IOMMU
1532 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), in drm_gpusvm_range_unmap_pages()
1535 if (ctx->in_notifier) in drm_gpusvm_range_unmap_pages()
1536 lockdep_assert_held_write(&gpusvm->notifier_lock); in drm_gpusvm_range_unmap_pages()
1542 if (!ctx->in_notifier) in drm_gpusvm_range_unmap_pages()
1548 * drm_gpusvm_migration_unlock_put_page() - Put a migration page
1560 * drm_gpusvm_migration_unlock_put_pages() - Put migration pages
1566 static void drm_gpusvm_migration_unlock_put_pages(unsigned long npages, in drm_gpusvm_migration_unlock_put_pages()
1567 unsigned long *migrate_pfn) in drm_gpusvm_migration_unlock_put_pages()
1569 unsigned long i; in drm_gpusvm_migration_unlock_put_pages()
1584 * drm_gpusvm_get_devmem_page() - Get a reference to a device memory page
1594 page->zone_device_data = drm_gpusvm_zdd_get(zdd); in drm_gpusvm_get_devmem_page()
1599 * drm_gpusvm_migrate_map_pages() - Map migration pages for GPU SVM migration
1611 * Return: 0 on success, -EFAULT if an error occurs during mapping.
1615 unsigned long *migrate_pfn, in drm_gpusvm_migrate_map_pages()
1616 unsigned long npages, in drm_gpusvm_migrate_map_pages()
1619 unsigned long i; in drm_gpusvm_migrate_map_pages()
1628 return -EFAULT; in drm_gpusvm_migrate_map_pages()
1632 return -EFAULT; in drm_gpusvm_migrate_map_pages()
1639 * drm_gpusvm_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration
1651 unsigned long npages, in drm_gpusvm_migrate_unmap_pages()
1654 unsigned long i; in drm_gpusvm_migrate_unmap_pages()
1665 * drm_gpusvm_migrate_to_devmem() - Migrate GPU SVM range to device memory
1670 * which should be dropped via ops->devmem_release or upon
1675 * performs the necessary setup and invokes the driver-specific operations for
1677 * safely reference @range until ops->devmem_release is called which only upon
1681 * Return: 0 on success, negative error code on failure.
1688 const struct drm_gpusvm_devmem_ops *ops = devmem_allocation->ops; in drm_gpusvm_migrate_to_devmem()
1689 unsigned long start = drm_gpusvm_range_start(range), in drm_gpusvm_migrate_to_devmem()
1694 .pgmap_owner = gpusvm->device_private_page_owner, in drm_gpusvm_migrate_to_devmem()
1697 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_migrate_to_devmem()
1698 unsigned long i, npages = npages_in_range(start, end); in drm_gpusvm_migrate_to_devmem()
1706 mmap_assert_locked(gpusvm->mm); in drm_gpusvm_migrate_to_devmem()
1708 if (!range->flags.migrate_devmem) in drm_gpusvm_migrate_to_devmem()
1709 return -EINVAL; in drm_gpusvm_migrate_to_devmem()
1711 if (!ops->populate_devmem_pfn || !ops->copy_to_devmem || in drm_gpusvm_migrate_to_devmem()
1712 !ops->copy_to_ram) in drm_gpusvm_migrate_to_devmem()
1713 return -EOPNOTSUPP; in drm_gpusvm_migrate_to_devmem()
1717 err = -ENOENT; in drm_gpusvm_migrate_to_devmem()
1721 if (end > vas->vm_end || start < vas->vm_start) { in drm_gpusvm_migrate_to_devmem()
1722 err = -EINVAL; in drm_gpusvm_migrate_to_devmem()
1727 err = -EBUSY; in drm_gpusvm_migrate_to_devmem()
1734 err = -ENOMEM; in drm_gpusvm_migrate_to_devmem()
1740 zdd = drm_gpusvm_zdd_alloc(gpusvm->device_private_page_owner); in drm_gpusvm_migrate_to_devmem()
1742 err = -ENOMEM; in drm_gpusvm_migrate_to_devmem()
1755 err = -EFAULT; in drm_gpusvm_migrate_to_devmem()
1760 err = -EBUSY; in drm_gpusvm_migrate_to_devmem()
1764 err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst); in drm_gpusvm_migrate_to_devmem()
1768 err = drm_gpusvm_migrate_map_pages(devmem_allocation->dev, dma_addr, in drm_gpusvm_migrate_to_devmem()
1781 err = ops->copy_to_devmem(pages, dma_addr, npages); in drm_gpusvm_migrate_to_devmem()
1786 devmem_allocation->timeslice_expiration = get_jiffies_64() + in drm_gpusvm_migrate_to_devmem()
1787 msecs_to_jiffies(ctx->timeslice_ms); in drm_gpusvm_migrate_to_devmem()
1788 zdd->devmem_allocation = devmem_allocation; /* Owns ref */ in drm_gpusvm_migrate_to_devmem()
1795 drm_gpusvm_migrate_unmap_pages(devmem_allocation->dev, dma_addr, npages, in drm_gpusvm_migrate_to_devmem()
1807 * drm_gpusvm_migrate_populate_ram_pfn() - Populate RAM PFNs for a VM area
1816 * This function populates the RAM migrate page frame numbers (PFNs) for the
1818 * RAM usage. If vas is non-NULL use alloc_page_vma for allocation, if NULL use
1821 * Return: 0 on success, negative error code on failure.
1825 unsigned long npages, in drm_gpusvm_migrate_populate_ram_pfn()
1826 unsigned long *mpages, in drm_gpusvm_migrate_populate_ram_pfn()
1827 unsigned long *src_mpfn, in drm_gpusvm_migrate_populate_ram_pfn()
1828 unsigned long *mpfn, in drm_gpusvm_migrate_populate_ram_pfn()
1829 unsigned long addr) in drm_gpusvm_migrate_populate_ram_pfn()
1831 unsigned long i; in drm_gpusvm_migrate_populate_ram_pfn()
1844 if (src_page->zone_device_data != in drm_gpusvm_migrate_populate_ram_pfn()
1845 fault_page->zone_device_data) in drm_gpusvm_migrate_populate_ram_pfn()
1882 return -ENOMEM; in drm_gpusvm_migrate_populate_ram_pfn()
1886 * drm_gpusvm_evict_to_ram() - Evict GPU SVM range to RAM
1892 * Return: 0 on success, negative error code on failure.
1896 const struct drm_gpusvm_devmem_ops *ops = devmem_allocation->ops; in drm_gpusvm_evict_to_ram()
1897 unsigned long npages, mpages = 0; in drm_gpusvm_evict_to_ram()
1899 unsigned long *src, *dst; in drm_gpusvm_evict_to_ram()
1905 npages = devmem_allocation->size >> PAGE_SHIFT; in drm_gpusvm_evict_to_ram()
1908 if (!mmget_not_zero(devmem_allocation->mm)) in drm_gpusvm_evict_to_ram()
1909 return -EFAULT; in drm_gpusvm_evict_to_ram()
1914 err = -ENOMEM; in drm_gpusvm_evict_to_ram()
1922 err = ops->populate_devmem_pfn(devmem_allocation, npages, src); in drm_gpusvm_evict_to_ram()
1935 err = drm_gpusvm_migrate_map_pages(devmem_allocation->dev, dma_addr, in drm_gpusvm_evict_to_ram()
1943 err = ops->copy_to_ram(pages, dma_addr, npages); in drm_gpusvm_evict_to_ram()
1952 drm_gpusvm_migrate_unmap_pages(devmem_allocation->dev, dma_addr, npages, in drm_gpusvm_evict_to_ram()
1957 mmput_async(devmem_allocation->mm); in drm_gpusvm_evict_to_ram()
1959 if (completion_done(&devmem_allocation->detached)) in drm_gpusvm_evict_to_ram()
1962 if (retry_count--) { in drm_gpusvm_evict_to_ram()
1967 return err ?: -EBUSY; in drm_gpusvm_evict_to_ram()
1972 * __drm_gpusvm_migrate_to_ram() - Migrate GPU SVM range to RAM (internal)
1980 * to RAM. It sets up the migration, populates + dma maps RAM PFNs, and
1981 * invokes the driver-specific operations for migration to RAM.
1983 * Return: 0 on success, negative error code on failure.
1988 unsigned long fault_addr, in __drm_gpusvm_migrate_to_ram()
1989 unsigned long size) in __drm_gpusvm_migrate_to_ram()
2001 unsigned long npages, mpages = 0; in __drm_gpusvm_migrate_to_ram()
2004 unsigned long start, end; in __drm_gpusvm_migrate_to_ram()
2009 zdd = page->zone_device_data; in __drm_gpusvm_migrate_to_ram()
2011 zdd->devmem_allocation->timeslice_expiration)) in __drm_gpusvm_migrate_to_ram()
2019 if (start < vas->vm_start) in __drm_gpusvm_migrate_to_ram()
2020 start = vas->vm_start; in __drm_gpusvm_migrate_to_ram()
2021 if (end > vas->vm_end) in __drm_gpusvm_migrate_to_ram()
2022 end = vas->vm_end; in __drm_gpusvm_migrate_to_ram()
2031 err = -ENOMEM; in __drm_gpusvm_migrate_to_ram()
2061 zdd = page->zone_device_data; in __drm_gpusvm_migrate_to_ram()
2062 ops = zdd->devmem_allocation->ops; in __drm_gpusvm_migrate_to_ram()
2063 dev = zdd->devmem_allocation->dev; in __drm_gpusvm_migrate_to_ram()
2079 err = ops->copy_to_ram(pages, dma_addr, npages); in __drm_gpusvm_migrate_to_ram()
2099 * drm_gpusvm_range_evict - Evict GPU SVM range
2105 * Return: 0 on success, a negative error code on failure.
2110 struct mmu_interval_notifier *notifier = &range->notifier->notifier; in drm_gpusvm_range_evict()
2118 unsigned long timeout = in drm_gpusvm_range_evict()
2120 unsigned long *pfns; in drm_gpusvm_range_evict()
2121 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), in drm_gpusvm_range_evict()
2124 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_evict()
2127 return -EFAULT; in drm_gpusvm_range_evict()
2131 return -ENOMEM; in drm_gpusvm_range_evict()
2137 err = -ETIME; in drm_gpusvm_range_evict()
2144 if (err != -EBUSY) in drm_gpusvm_range_evict()
2156 * drm_gpusvm_page_free() - Put GPU SVM zone device data associated with a page
2164 drm_gpusvm_zdd_put(page->zone_device_data); in drm_gpusvm_page_free()
2168 * drm_gpusvm_migrate_to_ram() - Migrate GPU SVM range to RAM (page fault handler)
2171 * This function is a page fault handler used to migrate a GPU SVM range to RAM.
2173 * the internal migration function to migrate the range back to RAM.
2179 struct drm_gpusvm_zdd *zdd = vmf->page->zone_device_data; in drm_gpusvm_migrate_to_ram()
2182 err = __drm_gpusvm_migrate_to_ram(vmf->vma, in drm_gpusvm_migrate_to_ram()
2183 zdd->device_private_page_owner, in drm_gpusvm_migrate_to_ram()
2184 vmf->page, vmf->address, in drm_gpusvm_migrate_to_ram()
2185 zdd->devmem_allocation->size); in drm_gpusvm_migrate_to_ram()
2191 * drm_gpusvm_pagemap_ops - Device page map operations for GPU SVM
2199 * drm_gpusvm_pagemap_ops_get() - Retrieve GPU SVM device page map operations
2210 * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range
2217 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, in drm_gpusvm_has_mapping()
2218 unsigned long end) in drm_gpusvm_has_mapping()
2234 * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
2244 lockdep_assert_held_write(&range->gpusvm->notifier_lock); in drm_gpusvm_range_set_unmapped()
2246 range->flags.unmapped = true; in drm_gpusvm_range_set_unmapped()
2247 if (drm_gpusvm_range_start(range) < mmu_range->start || in drm_gpusvm_range_set_unmapped()
2248 drm_gpusvm_range_end(range) > mmu_range->end) in drm_gpusvm_range_set_unmapped()
2249 range->flags.partial_unmap = true; in drm_gpusvm_range_set_unmapped()
2254 * drm_gpusvm_devmem_init() - Initialize a GPU SVM device memory allocation
2267 init_completion(&devmem_allocation->detached); in drm_gpusvm_devmem_init()
2268 devmem_allocation->dev = dev; in drm_gpusvm_devmem_init()
2269 devmem_allocation->mm = mm; in drm_gpusvm_devmem_init()
2270 devmem_allocation->ops = ops; in drm_gpusvm_devmem_init()
2271 devmem_allocation->dpagemap = dpagemap; in drm_gpusvm_devmem_init()
2272 devmem_allocation->size = size; in drm_gpusvm_devmem_init()