Lines Matching full:vrm
842 /* Set vrm->delta to the difference in VMA size specified by user. */
843 static void vrm_set_delta(struct vma_remap_struct *vrm) in vrm_set_delta() argument
845 vrm->delta = abs_diff(vrm->old_len, vrm->new_len); in vrm_set_delta()
849 static enum mremap_type vrm_remap_type(struct vma_remap_struct *vrm) in vrm_remap_type() argument
851 if (vrm->delta == 0) in vrm_remap_type()
854 if (vrm->old_len > vrm->new_len) in vrm_remap_type()
861 * When moving a VMA to vrm->new_adr, does this result in the new and old VMAs
864 static bool vrm_overlaps(struct vma_remap_struct *vrm) in vrm_overlaps() argument
866 unsigned long start_old = vrm->addr; in vrm_overlaps()
867 unsigned long start_new = vrm->new_addr; in vrm_overlaps()
868 unsigned long end_old = vrm->addr + vrm->old_len; in vrm_overlaps()
869 unsigned long end_new = vrm->new_addr + vrm->new_len; in vrm_overlaps()
888 static bool vrm_implies_new_addr(struct vma_remap_struct *vrm) in vrm_implies_new_addr() argument
890 return vrm->flags & (MREMAP_FIXED | MREMAP_DONTUNMAP); in vrm_implies_new_addr()
894 * Find an unmapped area for the requested vrm->new_addr.
900 * Returns 0 on success (with vrm->new_addr updated), or an error code upon
903 static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) in vrm_set_new_addr() argument
905 struct vm_area_struct *vma = vrm->vma; in vrm_set_new_addr()
908 pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; in vrm_set_new_addr()
910 unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; in vrm_set_new_addr()
913 if (vrm->flags & MREMAP_FIXED) in vrm_set_new_addr()
918 res = get_unmapped_area(vma->vm_file, new_addr, vrm->new_len, pgoff, in vrm_set_new_addr()
923 vrm->new_addr = res; in vrm_set_new_addr()
933 static bool vrm_charge(struct vma_remap_struct *vrm) in vrm_charge() argument
937 if (!(vrm->vma->vm_flags & VM_ACCOUNT)) in vrm_charge()
944 if (vrm->flags & MREMAP_DONTUNMAP) in vrm_charge()
945 charged = vrm->new_len >> PAGE_SHIFT; in vrm_charge()
947 charged = vrm->delta >> PAGE_SHIFT; in vrm_charge()
954 vrm->charged = charged; in vrm_charge()
959 * an error has occurred so we will not be using vrm->charged memory. Unaccount
962 static void vrm_uncharge(struct vma_remap_struct *vrm) in vrm_uncharge() argument
964 if (!(vrm->vma->vm_flags & VM_ACCOUNT)) in vrm_uncharge()
967 vm_unacct_memory(vrm->charged); in vrm_uncharge()
968 vrm->charged = 0; in vrm_uncharge()
973 * account for 'bytes' memory used, and if locked, indicate this in the VRM so
976 static void vrm_stat_account(struct vma_remap_struct *vrm, in vrm_stat_account() argument
981 struct vm_area_struct *vma = vrm->vma; in vrm_stat_account()
986 vrm->mlocked = true; in vrm_stat_account()
994 static unsigned long prep_move_vma(struct vma_remap_struct *vrm) in prep_move_vma() argument
997 struct vm_area_struct *vma = vrm->vma; in prep_move_vma()
998 unsigned long old_addr = vrm->addr; in prep_move_vma()
999 unsigned long old_len = vrm->old_len; in prep_move_vma()
1041 static void unmap_source_vma(struct vma_remap_struct *vrm) in unmap_source_vma() argument
1044 unsigned long addr = vrm->addr; in unmap_source_vma()
1045 unsigned long len = vrm->old_len; in unmap_source_vma()
1046 struct vm_area_struct *vma = vrm->vma; in unmap_source_vma()
1061 !(vrm->flags & MREMAP_DONTUNMAP); in unmap_source_vma()
1085 err = do_vmi_munmap(&vmi, mm, addr, len, vrm->uf_unmap, /* unlock= */false); in unmap_source_vma()
1086 vrm->vma = NULL; /* Invalidated. */ in unmap_source_vma()
1136 * Copy vrm->vma over to vrm->new_addr possibly adjusting size as part of the
1138 * where we reset vrm state to cause unmapping of the new VMA.
1143 static int copy_vma_and_data(struct vma_remap_struct *vrm, in copy_vma_and_data() argument
1146 unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; in copy_vma_and_data()
1148 unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; in copy_vma_and_data()
1150 struct vm_area_struct *vma = vrm->vma; in copy_vma_and_data()
1153 PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); in copy_vma_and_data()
1155 new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, in copy_vma_and_data()
1158 vrm_uncharge(vrm); in copy_vma_and_data()
1162 vrm->vma = vma; in copy_vma_and_data()
1167 if (moved_len < vrm->old_len) in copy_vma_and_data()
1173 PAGETABLE_MOVE(pmc_revert, new_vma, vma, vrm->new_addr, in copy_vma_and_data()
1174 vrm->addr, moved_len); in copy_vma_and_data()
1184 vrm->vma = new_vma; in copy_vma_and_data()
1185 vrm->old_len = vrm->new_len; in copy_vma_and_data()
1186 vrm->addr = vrm->new_addr; in copy_vma_and_data()
1188 mremap_userfaultfd_prep(new_vma, vrm->uf); in copy_vma_and_data()
1207 static void dontunmap_complete(struct vma_remap_struct *vrm, in dontunmap_complete() argument
1210 unsigned long start = vrm->addr; in dontunmap_complete()
1211 unsigned long end = vrm->addr + vrm->old_len; in dontunmap_complete()
1212 unsigned long old_start = vrm->vma->vm_start; in dontunmap_complete()
1213 unsigned long old_end = vrm->vma->vm_end; in dontunmap_complete()
1219 vm_flags_clear(vrm->vma, VM_LOCKED_MASK | VM_ACCOUNT); in dontunmap_complete()
1225 if (new_vma != vrm->vma && start == old_start && end == old_end) in dontunmap_complete()
1226 unlink_anon_vmas(vrm->vma); in dontunmap_complete()
1231 static unsigned long move_vma(struct vma_remap_struct *vrm) in move_vma() argument
1238 err = prep_move_vma(vrm); in move_vma()
1243 if (!vrm_charge(vrm)) in move_vma()
1247 vma_start_write(vrm->vma); in move_vma()
1250 err = copy_vma_and_data(vrm, &new_vma); in move_vma()
1270 vrm_stat_account(vrm, vrm->new_len); in move_vma()
1271 if (unlikely(!err && (vrm->flags & MREMAP_DONTUNMAP))) in move_vma()
1272 dontunmap_complete(vrm, new_vma); in move_vma()
1274 unmap_source_vma(vrm); in move_vma()
1278 return err ? (unsigned long)err : vrm->new_addr; in move_vma()
1287 static int resize_is_valid(struct vma_remap_struct *vrm) in resize_is_valid() argument
1290 struct vm_area_struct *vma = vrm->vma; in resize_is_valid()
1291 unsigned long addr = vrm->addr; in resize_is_valid()
1292 unsigned long old_len = vrm->old_len; in resize_is_valid()
1293 unsigned long new_len = vrm->new_len; in resize_is_valid()
1310 if ((vrm->flags & MREMAP_DONTUNMAP) && in resize_is_valid()
1330 if (!mlock_future_ok(mm, vma->vm_flags, vrm->delta)) in resize_is_valid()
1333 if (!may_expand_vm(mm, vma->vm_flags, vrm->delta >> PAGE_SHIFT)) in resize_is_valid()
1344 * then load the correct VMA into vrm->vma afterwards.
1346 static unsigned long shrink_vma(struct vma_remap_struct *vrm, in shrink_vma() argument
1350 unsigned long unmap_start = vrm->addr + vrm->new_len; in shrink_vma()
1351 unsigned long unmap_bytes = vrm->delta; in shrink_vma()
1355 VM_BUG_ON(vrm->remap_type != MREMAP_SHRINK); in shrink_vma()
1358 vrm->uf_unmap, drop_lock); in shrink_vma()
1359 vrm->vma = NULL; /* Invalidated. */ in shrink_vma()
1369 vrm->mmap_locked = false; in shrink_vma()
1371 vrm->vma = vma_lookup(mm, vrm->addr); in shrink_vma()
1372 if (!vrm->vma) in shrink_vma()
1383 static unsigned long mremap_to(struct vma_remap_struct *vrm) in mremap_to() argument
1389 if (vrm->new_len > TASK_SIZE || in mremap_to()
1390 vrm->new_addr > TASK_SIZE - vrm->new_len) in mremap_to()
1393 if (vrm_overlaps(vrm)) in mremap_to()
1396 if (vrm->flags & MREMAP_FIXED) { in mremap_to()
1402 err = do_munmap(mm, vrm->new_addr, vrm->new_len, in mremap_to()
1403 vrm->uf_unmap_early); in mremap_to()
1404 vrm->vma = NULL; /* Invalidated. */ in mremap_to()
1412 vrm->vma = vma_lookup(mm, vrm->addr); in mremap_to()
1413 if (!vrm->vma) in mremap_to()
1417 if (vrm->remap_type == MREMAP_SHRINK) { in mremap_to()
1418 err = shrink_vma(vrm, /* drop_lock= */false); in mremap_to()
1423 vrm->old_len = vrm->new_len; in mremap_to()
1426 err = resize_is_valid(vrm); in mremap_to()
1431 if (vrm->flags & MREMAP_DONTUNMAP) { in mremap_to()
1432 vm_flags_t vm_flags = vrm->vma->vm_flags; in mremap_to()
1433 unsigned long pages = vrm->old_len >> PAGE_SHIFT; in mremap_to()
1439 err = vrm_set_new_addr(vrm); in mremap_to()
1443 return move_vma(vrm); in mremap_to()
1461 static bool vrm_can_expand_in_place(struct vma_remap_struct *vrm) in vrm_can_expand_in_place() argument
1463 /* Number of bytes from vrm->addr to end of VMA. */ in vrm_can_expand_in_place()
1464 unsigned long suffix_bytes = vrm->vma->vm_end - vrm->addr; in vrm_can_expand_in_place()
1467 if (suffix_bytes != vrm->old_len) in vrm_can_expand_in_place()
1471 if (!vma_expandable(vrm->vma, vrm->delta)) in vrm_can_expand_in_place()
1481 static unsigned long check_mremap_params(struct vma_remap_struct *vrm) in check_mremap_params() argument
1484 unsigned long addr = vrm->addr; in check_mremap_params()
1485 unsigned long flags = vrm->flags; in check_mremap_params()
1500 if (!PAGE_ALIGN(vrm->new_len)) in check_mremap_params()
1504 if (!vrm_implies_new_addr(vrm)) in check_mremap_params()
1508 if (offset_in_page(vrm->new_addr)) in check_mremap_params()
1516 if (flags & MREMAP_DONTUNMAP && vrm->old_len != vrm->new_len) in check_mremap_params()
1545 static unsigned long expand_vma_in_place(struct vma_remap_struct *vrm) in expand_vma_in_place() argument
1548 struct vm_area_struct *vma = vrm->vma; in expand_vma_in_place()
1551 if (!vrm_charge(vrm)) in expand_vma_in_place()
1563 vma = vma_merge_extend(&vmi, vma, vrm->delta); in expand_vma_in_place()
1565 vrm_uncharge(vrm); in expand_vma_in_place()
1568 vrm->vma = vma; in expand_vma_in_place()
1570 vrm_stat_account(vrm, vrm->delta); in expand_vma_in_place()
1575 static bool align_hugetlb(struct vma_remap_struct *vrm) in align_hugetlb() argument
1577 struct hstate *h __maybe_unused = hstate_vma(vrm->vma); in align_hugetlb()
1579 vrm->old_len = ALIGN(vrm->old_len, huge_page_size(h)); in align_hugetlb()
1580 vrm->new_len = ALIGN(vrm->new_len, huge_page_size(h)); in align_hugetlb()
1583 if (vrm->addr & ~huge_page_mask(h)) in align_hugetlb()
1585 if (vrm->new_addr & ~huge_page_mask(h)) in align_hugetlb()
1592 if (vrm->new_len > vrm->old_len) in align_hugetlb()
1595 vrm_set_delta(vrm); in align_hugetlb()
1607 static unsigned long expand_vma(struct vma_remap_struct *vrm) in expand_vma() argument
1610 unsigned long addr = vrm->addr; in expand_vma()
1612 err = resize_is_valid(vrm); in expand_vma()
1620 if (vrm_can_expand_in_place(vrm)) { in expand_vma()
1621 err = expand_vma_in_place(vrm); in expand_vma()
1630 if (vrm->mlocked) in expand_vma()
1631 vrm->new_addr = addr; in expand_vma()
1643 if (!(vrm->flags & MREMAP_MAYMOVE)) in expand_vma()
1647 err = vrm_set_new_addr(vrm); in expand_vma()
1651 return move_vma(vrm); in expand_vma()
1658 static unsigned long mremap_at(struct vma_remap_struct *vrm) in mremap_at() argument
1662 switch (vrm->remap_type) { in mremap_at()
1667 return vrm->addr; in mremap_at()
1676 res = shrink_vma(vrm, /* drop_lock= */true); in mremap_at()
1680 return vrm->addr; in mremap_at()
1682 return expand_vma(vrm); in mremap_at()
1688 static unsigned long do_mremap(struct vma_remap_struct *vrm) in do_mremap() argument
1694 ret = check_mremap_params(vrm); in do_mremap()
1698 vrm->old_len = PAGE_ALIGN(vrm->old_len); in do_mremap()
1699 vrm->new_len = PAGE_ALIGN(vrm->new_len); in do_mremap()
1700 vrm_set_delta(vrm); in do_mremap()
1704 vrm->mmap_locked = true; in do_mremap()
1706 vma = vrm->vma = vma_lookup(mm, vrm->addr); in do_mremap()
1719 if (is_vm_hugetlb_page(vma) && !align_hugetlb(vrm)) { in do_mremap()
1724 vrm->remap_type = vrm_remap_type(vrm); in do_mremap()
1727 ret = vrm_implies_new_addr(vrm) ? mremap_to(vrm) : mremap_at(vrm); in do_mremap()
1730 if (vrm->mmap_locked) { in do_mremap()
1732 vrm->mmap_locked = false; in do_mremap()
1734 if (!offset_in_page(ret) && vrm->mlocked && vrm->new_len > vrm->old_len) in do_mremap()
1735 mm_populate(vrm->new_addr + vrm->old_len, vrm->delta); in do_mremap()
1738 userfaultfd_unmap_complete(mm, vrm->uf_unmap_early); in do_mremap()
1739 mremap_userfaultfd_complete(vrm->uf, vrm->addr, ret, vrm->old_len); in do_mremap()
1740 userfaultfd_unmap_complete(mm, vrm->uf_unmap); in do_mremap()
1770 struct vma_remap_struct vrm = { in SYSCALL_DEFINE5() local
1784 return do_mremap(&vrm); in SYSCALL_DEFINE5()