Lines Matching refs:dmirror
83 struct dmirror *dmirror; member
90 struct dmirror { struct
149 dmirror_select_device(struct dmirror *dmirror) in dmirror_select_device() argument
151 return (dmirror->mdevice->zone_device_type == in dmirror_select_device()
165 struct dmirror *dmirror; in dmirror_fops_open() local
169 dmirror = kzalloc_obj(*dmirror); in dmirror_fops_open()
170 if (dmirror == NULL) in dmirror_fops_open()
173 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); in dmirror_fops_open()
174 mutex_init(&dmirror->mutex); in dmirror_fops_open()
175 xa_init(&dmirror->pt); in dmirror_fops_open()
177 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, in dmirror_fops_open()
180 kfree(dmirror); in dmirror_fops_open()
184 filp->private_data = dmirror; in dmirror_fops_open()
190 struct dmirror *dmirror = filp->private_data; in dmirror_fops_release() local
192 mmu_interval_notifier_remove(&dmirror->notifier); in dmirror_fops_release()
193 xa_destroy(&dmirror->pt); in dmirror_fops_release()
194 kfree(dmirror); in dmirror_fops_release()
210 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) in dmirror_do_fault() argument
236 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); in dmirror_do_fault()
244 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, in dmirror_do_update() argument
255 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, in dmirror_do_update()
257 xa_erase(&dmirror->pt, pfn); in dmirror_do_update()
264 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); in dmirror_interval_invalidate() local
271 range->owner == dmirror->mdevice) in dmirror_interval_invalidate()
275 mutex_lock(&dmirror->mutex); in dmirror_interval_invalidate()
276 else if (!mutex_trylock(&dmirror->mutex)) in dmirror_interval_invalidate()
280 dmirror_do_update(dmirror, range->start, range->end); in dmirror_interval_invalidate()
282 mutex_unlock(&dmirror->mutex); in dmirror_interval_invalidate()
290 static int dmirror_range_fault(struct dmirror *dmirror, in dmirror_range_fault() argument
293 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_range_fault()
314 mutex_lock(&dmirror->mutex); in dmirror_range_fault()
317 mutex_unlock(&dmirror->mutex); in dmirror_range_fault()
323 ret = dmirror_do_fault(dmirror, range); in dmirror_range_fault()
325 mutex_unlock(&dmirror->mutex); in dmirror_range_fault()
330 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, in dmirror_fault() argument
333 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_fault()
337 .notifier = &dmirror->notifier, in dmirror_fault()
342 .dev_private_owner = dmirror->mdevice, in dmirror_fault()
354 ret = dmirror_range_fault(dmirror, &range); in dmirror_fault()
363 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, in dmirror_do_read() argument
375 entry = xa_load(&dmirror->pt, pfn); in dmirror_do_read()
389 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) in dmirror_read() argument
406 mutex_lock(&dmirror->mutex); in dmirror_read()
407 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_read()
408 mutex_unlock(&dmirror->mutex); in dmirror_read()
413 ret = dmirror_fault(dmirror, start, end, false); in dmirror_read()
429 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, in dmirror_do_write() argument
441 entry = xa_load(&dmirror->pt, pfn); in dmirror_do_write()
455 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) in dmirror_write() argument
477 mutex_lock(&dmirror->mutex); in dmirror_write()
478 ret = dmirror_do_write(dmirror, start, end, &bounce); in dmirror_write()
479 mutex_unlock(&dmirror->mutex); in dmirror_write()
484 ret = dmirror_fault(dmirror, start, end, true); in dmirror_write()
628 static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, in dmirror_devmem_alloc_page() argument
634 struct dmirror_device *mdevice = dmirror->mdevice; in dmirror_devmem_alloc_page()
678 struct dmirror *dmirror) in dmirror_migrate_alloc_and_copy() argument
705 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { in dmirror_migrate_alloc_and_copy()
706 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; in dmirror_migrate_alloc_and_copy()
709 dpage = dmirror_devmem_alloc_page(dmirror, is_large); in dmirror_migrate_alloc_and_copy()
728 dpage = dmirror_devmem_alloc_page(dmirror, false); in dmirror_migrate_alloc_and_copy()
730 rpage->zone_device_data = dmirror; in dmirror_migrate_alloc_and_copy()
754 rpage->zone_device_data = dmirror; in dmirror_migrate_alloc_and_copy()
797 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, in dmirror_check_atomic() argument
805 entry = xa_load(&dmirror->pt, pfn); in dmirror_check_atomic()
814 struct dmirror *dmirror) in dmirror_atomic_map() argument
819 mutex_lock(&dmirror->mutex); in dmirror_atomic_map()
822 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); in dmirror_atomic_map()
824 mutex_unlock(&dmirror->mutex); in dmirror_atomic_map()
828 mutex_unlock(&dmirror->mutex); in dmirror_atomic_map()
833 struct dmirror *dmirror) in dmirror_migrate_finalize_and_map() argument
844 mutex_lock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
873 entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); in dmirror_migrate_finalize_and_map()
875 mutex_unlock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
881 mutex_unlock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
885 static int dmirror_exclusive(struct dmirror *dmirror, in dmirror_exclusive() argument
890 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_exclusive()
914 ret = dmirror_atomic_map(addr, page, dmirror); in dmirror_exclusive()
928 mutex_lock(&dmirror->mutex); in dmirror_exclusive()
929 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_exclusive()
930 mutex_unlock(&dmirror->mutex); in dmirror_exclusive()
943 struct dmirror *dmirror) in dmirror_devmem_fault_alloc_and_copy() argument
975 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { in dmirror_devmem_fault_alloc_and_copy()
976 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; in dmirror_devmem_fault_alloc_and_copy()
1014 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); in dmirror_devmem_fault_alloc_and_copy()
1039 static int dmirror_migrate_to_system(struct dmirror *dmirror, in dmirror_migrate_to_system() argument
1044 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_migrate_to_system()
1081 args.pgmap_owner = dmirror->mdevice; in dmirror_migrate_to_system()
1082 args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND; in dmirror_migrate_to_system()
1089 dmirror_devmem_fault_alloc_and_copy(&args, dmirror); in dmirror_migrate_to_system()
1104 static int dmirror_migrate_to_device(struct dmirror *dmirror, in dmirror_migrate_to_device() argument
1109 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_migrate_to_device()
1155 args.pgmap_owner = dmirror->mdevice; in dmirror_migrate_to_device()
1163 dmirror_migrate_alloc_and_copy(&args, dmirror); in dmirror_migrate_to_device()
1165 dmirror_migrate_finalize_and_map(&args, dmirror); in dmirror_migrate_to_device()
1178 mutex_lock(&dmirror->mutex); in dmirror_migrate_to_device()
1179 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_migrate_to_device()
1180 mutex_unlock(&dmirror->mutex); in dmirror_migrate_to_device()
1199 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, in dmirror_mkentry() argument
1216 if (dmirror->mdevice == dmirror_page_to_device(page)) in dmirror_mkentry()
1222 if (dmirror->mdevice == dmirror_page_to_device(page)) in dmirror_mkentry()
1246 struct dmirror *dmirror = dmi->dmirror; in dmirror_snapshot_invalidate() local
1249 mutex_lock(&dmirror->mutex); in dmirror_snapshot_invalidate()
1250 else if (!mutex_trylock(&dmirror->mutex)) in dmirror_snapshot_invalidate()
1259 mutex_unlock(&dmirror->mutex); in dmirror_snapshot_invalidate()
1267 static int dmirror_range_snapshot(struct dmirror *dmirror, in dmirror_range_snapshot() argument
1271 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_range_snapshot()
1279 notifier.dmirror = dmirror; in dmirror_range_snapshot()
1305 mutex_lock(&dmirror->mutex); in dmirror_range_snapshot()
1308 mutex_unlock(&dmirror->mutex); in dmirror_range_snapshot()
1316 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); in dmirror_range_snapshot()
1318 mutex_unlock(&dmirror->mutex); in dmirror_range_snapshot()
1324 static int dmirror_snapshot(struct dmirror *dmirror, in dmirror_snapshot() argument
1327 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_snapshot()
1337 .dev_private_owner = dmirror->mdevice, in dmirror_snapshot()
1362 ret = dmirror_range_snapshot(dmirror, &range, perm); in dmirror_snapshot()
1484 struct dmirror *dmirror; in dmirror_fops_unlocked_ioctl() local
1487 dmirror = filp->private_data; in dmirror_fops_unlocked_ioctl()
1488 if (!dmirror) in dmirror_fops_unlocked_ioctl()
1504 ret = dmirror_read(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1508 ret = dmirror_write(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1512 ret = dmirror_migrate_to_device(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1516 ret = dmirror_migrate_to_system(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1520 ret = dmirror_exclusive(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1524 ret = dmirror_check_atomic(dmirror, cmd.addr, in dmirror_fops_unlocked_ioctl()
1529 ret = dmirror_snapshot(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1533 dmirror_device_remove_chunks(dmirror->mdevice); in dmirror_fops_unlocked_ioctl()
1537 dmirror->flags = cmd.npages; in dmirror_fops_unlocked_ioctl()
1622 struct dmirror *dmirror; in dmirror_devmem_fault() local
1632 dmirror = rpage->zone_device_data; in dmirror_devmem_fault()
1658 args.pgmap_owner = dmirror->mdevice; in dmirror_devmem_fault()
1659 args.flags = dmirror_select_device(dmirror); in dmirror_devmem_fault()
1673 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); in dmirror_devmem_fault()