Lines Matching refs:map
129 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
132 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
133 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
134 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
136 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
141 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
144 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
166 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
168 if (start < vm_map_min(map)) \
169 start = vm_map_min(map); \
170 if (end > vm_map_max(map)) \
171 end = vm_map_max(map); \
291 vm_map_t map; in vmspace_zinit() local
294 map = &vm->vm_map; in vmspace_zinit()
296 memset(map, 0, sizeof(*map)); /* set MAP_SYSTEM_MAP to false */ in vmspace_zinit()
297 sx_init(&map->lock, "vm map (user)"); in vmspace_zinit()
524 _vm_map_lock(vm_map_t map, const char *file, int line) in _vm_map_lock() argument
527 if (vm_map_is_system(map)) in _vm_map_lock()
528 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock()
530 sx_xlock_(&map->lock, file, line); in _vm_map_lock()
531 map->timestamp++; in _vm_map_lock()
615 _vm_map_assert_locked(vm_map_t map, const char *file, int line) in _vm_map_assert_locked() argument
618 if (vm_map_is_system(map)) in _vm_map_assert_locked()
619 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_assert_locked()
621 sx_assert_(&map->lock, SA_XLOCKED, file, line); in _vm_map_assert_locked()
624 #define VM_MAP_ASSERT_LOCKED(map) \ argument
625 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
636 static void _vm_map_assert_consistent(vm_map_t map, int check);
638 #define VM_MAP_ASSERT_CONSISTENT(map) \ argument
639 _vm_map_assert_consistent(map, VMMAP_CHECK_ALL)
641 #define VM_MAP_UNLOCK_CONSISTENT(map) do { \ argument
642 if (map->nupdates > map->nentries) { \
643 _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \
644 map->nupdates = 0; \
648 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
651 #define VM_MAP_ASSERT_LOCKED(map) argument
652 #define VM_MAP_ASSERT_CONSISTENT(map) argument
653 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
657 _vm_map_unlock(vm_map_t map, const char *file, int line) in _vm_map_unlock() argument
660 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock()
661 if (vm_map_is_system(map)) { in _vm_map_unlock()
663 if (map == kernel_map && (map->flags & MAP_REPLENISH) != 0) { in _vm_map_unlock()
665 map->flags &= ~MAP_REPLENISH; in _vm_map_unlock()
668 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock()
670 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock()
676 _vm_map_lock_read(vm_map_t map, const char *file, int line) in _vm_map_lock_read() argument
679 if (vm_map_is_system(map)) in _vm_map_lock_read()
680 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock_read()
682 sx_slock_(&map->lock, file, line); in _vm_map_lock_read()
686 _vm_map_unlock_read(vm_map_t map, const char *file, int line) in _vm_map_unlock_read() argument
689 if (vm_map_is_system(map)) { in _vm_map_unlock_read()
690 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_read()
692 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_read()
694 sx_sunlock_(&map->lock, file, line); in _vm_map_unlock_read()
700 _vm_map_trylock(vm_map_t map, const char *file, int line) in _vm_map_trylock() argument
704 error = vm_map_is_system(map) ? in _vm_map_trylock()
705 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock()
706 !sx_try_xlock_(&map->lock, file, line); in _vm_map_trylock()
708 map->timestamp++; in _vm_map_trylock()
713 _vm_map_trylock_read(vm_map_t map, const char *file, int line) in _vm_map_trylock_read() argument
717 error = vm_map_is_system(map) ? in _vm_map_trylock_read()
718 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock_read()
719 !sx_try_slock_(&map->lock, file, line); in _vm_map_trylock_read()
734 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_upgrade() argument
738 if (vm_map_is_system(map)) { in _vm_map_lock_upgrade()
739 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_upgrade()
741 if (!sx_try_upgrade_(&map->lock, file, line)) { in _vm_map_lock_upgrade()
742 last_timestamp = map->timestamp; in _vm_map_lock_upgrade()
743 sx_sunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
749 sx_xlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
750 if (last_timestamp != map->timestamp) { in _vm_map_lock_upgrade()
751 sx_xunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
756 map->timestamp++; in _vm_map_lock_upgrade()
761 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_downgrade() argument
764 if (vm_map_is_system(map)) { in _vm_map_lock_downgrade()
765 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_lock_downgrade()
767 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_downgrade()
769 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_lock_downgrade()
770 sx_downgrade_(&map->lock, file, line); in _vm_map_lock_downgrade()
781 vm_map_locked(vm_map_t map) in vm_map_locked() argument
784 if (vm_map_is_system(map)) in vm_map_locked()
785 return (mtx_owned(&map->system_mtx)); in vm_map_locked()
786 return (sx_xlocked(&map->lock)); in vm_map_locked()
804 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) in _vm_map_unlock_and_wait() argument
807 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock_and_wait()
809 if (vm_map_is_system(map)) { in _vm_map_unlock_and_wait()
810 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_and_wait()
812 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_and_wait()
814 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock_and_wait()
816 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", in _vm_map_unlock_and_wait()
827 vm_map_wakeup(vm_map_t map) in vm_map_wakeup() argument
837 wakeup(&map->root); in vm_map_wakeup()
841 vm_map_busy(vm_map_t map) in vm_map_busy() argument
844 VM_MAP_ASSERT_LOCKED(map); in vm_map_busy()
845 map->busy++; in vm_map_busy()
849 vm_map_unbusy(vm_map_t map) in vm_map_unbusy() argument
852 VM_MAP_ASSERT_LOCKED(map); in vm_map_unbusy()
853 KASSERT(map->busy, ("vm_map_unbusy: not busy")); in vm_map_unbusy()
854 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { in vm_map_unbusy()
855 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); in vm_map_unbusy()
856 wakeup(&map->busy); in vm_map_unbusy()
861 vm_map_wait_busy(vm_map_t map) in vm_map_wait_busy() argument
864 VM_MAP_ASSERT_LOCKED(map); in vm_map_wait_busy()
865 while (map->busy) { in vm_map_wait_busy()
866 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); in vm_map_wait_busy()
867 if (vm_map_is_system(map)) in vm_map_wait_busy()
868 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); in vm_map_wait_busy()
870 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); in vm_map_wait_busy()
872 map->timestamp++; in vm_map_wait_busy()
886 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in _vm_map_init() argument
889 map->header.eflags = MAP_ENTRY_HEADER; in _vm_map_init()
890 map->pmap = pmap; in _vm_map_init()
891 map->header.end = min; in _vm_map_init()
892 map->header.start = max; in _vm_map_init()
893 map->flags = 0; in _vm_map_init()
894 map->header.left = map->header.right = &map->header; in _vm_map_init()
895 map->root = NULL; in _vm_map_init()
896 map->timestamp = 0; in _vm_map_init()
897 map->busy = 0; in _vm_map_init()
898 map->anon_loc = 0; in _vm_map_init()
900 map->nupdates = 0; in _vm_map_init()
905 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init() argument
907 _vm_map_init(map, pmap, min, max); in vm_map_init()
908 sx_init(&map->lock, "vm map (user)"); in vm_map_init()
912 vm_map_init_system(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init_system() argument
914 _vm_map_init(map, pmap, min, max); in vm_map_init_system()
915 vm_map_modflags(map, MAP_SYSTEM_MAP, 0); in vm_map_init_system()
916 mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | in vm_map_init_system()
926 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
928 uma_zfree(vm_map_is_system(map) ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
938 vm_map_entry_create(vm_map_t map) in vm_map_entry_create() argument
943 if (map == kernel_map) { in vm_map_entry_create()
944 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_create()
962 if (vm_map_is_system(map)) { in vm_map_entry_create()
1129 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, in vm_map_splay_split() argument
1134 left = right = &map->header; in vm_map_splay_split()
1135 root = map->root; in vm_map_splay_split()
1338 vm_map_splay(vm_map_t map, vm_offset_t addr) in vm_map_splay() argument
1343 header = &map->header; in vm_map_splay()
1344 root = vm_map_splay_split(map, addr, 0, &llist, &rlist); in vm_map_splay()
1371 map->root = root; in vm_map_splay()
1372 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_splay()
1386 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1392 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1393 map->nentries, entry); in vm_map_entry_link()
1394 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_link()
1395 map->nentries++; in vm_map_entry_link()
1396 header = &map->header; in vm_map_entry_link()
1397 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1441 map->root = entry; in vm_map_entry_link()
1442 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_link()
1451 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1457 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unlink()
1458 header = &map->header; in vm_map_entry_unlink()
1459 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1486 map->root = root; in vm_map_entry_unlink()
1487 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_unlink()
1488 map->nentries--; in vm_map_entry_unlink()
1489 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1490 map->nentries, entry); in vm_map_entry_unlink()
1502 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) in vm_map_entry_resize() argument
1506 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_resize()
1507 header = &map->header; in vm_map_entry_resize()
1508 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize()
1515 map->root = root; in vm_map_entry_resize()
1516 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_resize()
1518 __func__, map, map->nentries, entry); in vm_map_entry_resize()
1533 vm_map_t map, in vm_map_lookup_entry() argument
1544 header = &map->header; in vm_map_lookup_entry()
1545 cur = map->root; in vm_map_lookup_entry()
1554 if ((locked = vm_map_locked(map)) || in vm_map_lookup_entry()
1555 sx_try_upgrade(&map->lock)) { in vm_map_lookup_entry()
1562 cur = vm_map_splay(map, address); in vm_map_lookup_entry()
1564 VM_MAP_UNLOCK_CONSISTENT(map); in vm_map_lookup_entry()
1565 sx_downgrade(&map->lock); in vm_map_lookup_entry()
1613 vm_map_insert1(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert1() argument
1625 VM_MAP_ASSERT_LOCKED(map); in vm_map_insert1()
1639 if (start == end || !vm_map_range_valid(map, start, end)) in vm_map_insert1()
1642 if ((map->flags & MAP_WXORX) != 0 && (prot & (VM_PROT_WRITE | in vm_map_insert1()
1650 if (vm_map_lookup_entry(map, start, &prev_entry)) in vm_map_insert1()
1731 if (map == kernel_map && end > kernel_vm_end) { in vm_map_insert1()
1775 map->size += end - prev_entry->end; in vm_map_insert1()
1776 vm_map_entry_resize(map, prev_entry, in vm_map_insert1()
1778 *res = vm_map_try_merge_entries(map, prev_entry, in vm_map_insert1()
1805 new_entry = vm_map_entry_create(map); in vm_map_insert1()
1829 vm_map_entry_link(map, new_entry); in vm_map_insert1()
1831 map->size += new_entry->end - new_entry->start; in vm_map_insert1()
1839 vm_map_try_merge_entries(map, prev_entry, new_entry); in vm_map_insert1()
1840 *res = vm_map_try_merge_entries(map, new_entry, next_entry); in vm_map_insert1()
1843 vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), in vm_map_insert1()
1862 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert() argument
1867 return (vm_map_insert1(map, object, offset, start, end, prot, max, in vm_map_insert()
1889 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) in vm_map_findspace() argument
1895 VM_MAP_ASSERT_LOCKED(map); in vm_map_findspace()
1901 start = MAX(start, vm_map_min(map)); in vm_map_findspace()
1902 if (start >= vm_map_max(map) || length > vm_map_max(map) - start) in vm_map_findspace()
1903 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1906 if (map->root == NULL) in vm_map_findspace()
1916 header = &map->header; in vm_map_findspace()
1917 root = vm_map_splay_split(map, start, length, &llist, &rlist); in vm_map_findspace()
1937 map->root = root; in vm_map_findspace()
1938 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1944 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1975 map->root = root; in vm_map_findspace()
1976 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1981 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_fixed() argument
1991 vm_map_lock(map); in vm_map_fixed()
1992 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_fixed()
1994 result = vm_map_delete(map, start, end); in vm_map_fixed()
1999 result = vm_map_stack_locked(map, start, length, sgrowsiz, in vm_map_fixed()
2002 result = vm_map_insert(map, object, offset, start, end, in vm_map_fixed()
2006 vm_map_unlock(map); in vm_map_fixed()
2058 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_alignspace() argument
2064 VM_MAP_ASSERT_LOCKED(map); in vm_map_alignspace()
2066 KASSERT(free_addr == vm_map_findspace(map, free_addr, length), in vm_map_alignspace()
2094 *addr = vm_map_findspace(map, aligned_addr, length); in vm_map_alignspace()
2095 if (*addr + length > vm_map_max(map) || in vm_map_alignspace()
2111 vm_map_find_aligned(vm_map_t map, vm_offset_t *addr, vm_size_t length, in vm_map_find_aligned() argument
2115 *addr = vm_map_findspace(map, *addr, length); in vm_map_find_aligned()
2116 if (*addr + length > vm_map_max(map) || in vm_map_find_aligned()
2119 return (vm_map_alignspace(map, NULL, 0, addr, length, max_addr, in vm_map_find_aligned()
2133 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find() argument
2140 vm_map_lock(map); in vm_map_find()
2141 rv = vm_map_find_locked(map, object, offset, addr, length, max_addr, in vm_map_find()
2143 vm_map_unlock(map); in vm_map_find()
2148 vm_map_find_locked(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_locked() argument
2169 en_aslr = (map->flags & MAP_ASLR) != 0; in vm_map_find_locked()
2171 (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && in vm_map_find_locked()
2178 (map->flags & MAP_ASLR_IGNSTART) != 0) in vm_map_find_locked()
2179 curr_min_addr = min_addr = vm_map_min(map); in vm_map_find_locked()
2182 curr_min_addr = map->anon_loc; in vm_map_find_locked()
2217 curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? in vm_map_find_locked()
2218 vm_map_min(map) : min_addr; in vm_map_find_locked()
2244 gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && in vm_map_find_locked()
2247 *addr = vm_map_findspace(map, curr_min_addr, in vm_map_find_locked()
2250 vm_map_max(map)) in vm_map_find_locked()
2257 *addr = vm_map_findspace(map, curr_min_addr, length); in vm_map_find_locked()
2258 if (*addr + length > vm_map_max(map) || in vm_map_find_locked()
2270 (rv = vm_map_alignspace(map, object, offset, addr, length, in vm_map_find_locked()
2282 if (!vm_map_range_valid(map, *addr, *addr + length)) in vm_map_find_locked()
2284 rv = vm_map_delete(map, *addr, *addr + length); in vm_map_find_locked()
2289 rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, in vm_map_find_locked()
2292 rv = vm_map_insert(map, object, offset, *addr, *addr + length, in vm_map_find_locked()
2301 if (update_anon && rv == KERN_SUCCESS && (map->anon_loc == 0 || in vm_map_find_locked()
2302 *addr < map->anon_loc)) in vm_map_find_locked()
2303 map->anon_loc = *addr; in vm_map_find_locked()
2320 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_min() argument
2334 rv = vm_map_find(map, object, offset, addr, length, max_addr, in vm_map_find_min()
2371 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2388 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2402 vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev_entry, in vm_map_try_merge_entries() argument
2406 VM_MAP_ASSERT_LOCKED(map); in vm_map_try_merge_entries()
2409 vm_map_entry_unlink(map, prev_entry, UNLINK_MERGE_NEXT); in vm_map_try_merge_entries()
2410 vm_map_merged_neighbor_dispose(map, prev_entry); in vm_map_try_merge_entries()
2444 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_charge_object() argument
2448 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_charge_object()
2452 if (object == NULL && !vm_map_is_system(map) && in vm_map_entry_charge_object()
2477 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_clone() argument
2481 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_clone()
2487 vm_map_entry_charge_object(map, entry); in vm_map_entry_clone()
2490 new_entry = vm_map_entry_create(map); in vm_map_entry_clone()
2515 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) in vm_map_clip_start() argument
2520 if (!vm_map_is_system(map)) in vm_map_clip_start()
2522 "%s: map %p entry %p start 0x%jx", __func__, map, entry, in vm_map_clip_start()
2528 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_start()
2538 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_start()
2545 vm_map_entry_link(map, new_entry); in vm_map_clip_start()
2557 vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, in vm_map_lookup_clip_start() argument
2563 if (!vm_map_is_system(map)) in vm_map_lookup_clip_start()
2565 "%s: map %p start 0x%jx prev %p", __func__, map, in vm_map_lookup_clip_start()
2568 if (vm_map_lookup_entry(map, start, prev_entry)) { in vm_map_lookup_clip_start()
2570 rv = vm_map_clip_start(map, entry, start); in vm_map_lookup_clip_start()
2588 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) in vm_map_clip_end() argument
2593 if (!vm_map_is_system(map)) in vm_map_clip_end()
2595 "%s: map %p entry %p end 0x%jx", __func__, map, entry, in vm_map_clip_end()
2601 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_end()
2611 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_end()
2618 vm_map_entry_link(map, new_entry); in vm_map_clip_end()
2643 vm_map_t map, in vm_map_submap() argument
2657 vm_map_lock(map); in vm_map_submap()
2658 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_submap()
2659 if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && in vm_map_submap()
2662 result = vm_map_clip_start(map, entry, start); in vm_map_submap()
2665 result = vm_map_clip_end(map, entry, end); in vm_map_submap()
2673 vm_map_unlock(map); in vm_map_submap()
2701 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, in vm_map_pmap_enter() argument
2715 pmap_object_init_pt(map->pmap, addr, object, pindex, in vm_map_pmap_enter()
2773 pmap_enter_object(map->pmap, start, addr + in vm_map_pmap_enter()
2779 pmap_enter_object(map->pmap, start, addr + ptoa(psize), in vm_map_pmap_enter()
2812 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_protect() argument
2834 vm_map_lock(map); in vm_map_protect()
2836 if ((map->flags & MAP_WXORX) != 0 && in vm_map_protect()
2839 vm_map_unlock(map); in vm_map_protect()
2849 vm_map_wait_busy(map); in vm_map_protect()
2851 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_protect()
2853 if (!vm_map_lookup_entry(map, start, &first_entry)) in vm_map_protect()
2867 first_entry != vm_map_entry_first(map)) in vm_map_protect()
2883 vm_map_unlock(map); in vm_map_protect()
2892 vm_map_unlock(map); in vm_map_protect()
2908 vm_map_unlock_and_wait(map, 0); in vm_map_protect()
2919 rv = vm_map_clip_start(map, first_entry, start); in vm_map_protect()
2921 vm_map_unlock(map); in vm_map_protect()
2926 rv = vm_map_clip_end(map, entry, end); in vm_map_protect()
2928 vm_map_unlock(map); in vm_map_protect()
2986 vm_map_try_merge_entries(map, prev_entry, entry), in vm_map_protect()
3015 vm_fault_copy_entry(map, map, entry, entry, NULL); in vm_map_protect()
3024 pmap_protect(map->pmap, entry->start, in vm_map_protect()
3030 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_protect()
3031 vm_map_unlock(map); in vm_map_protect()
3045 vm_map_t map, in vm_map_madvise() argument
3071 vm_map_lock(map); in vm_map_madvise()
3079 vm_map_lock_read(map); in vm_map_madvise()
3088 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_madvise()
3097 rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); in vm_map_madvise()
3099 vm_map_unlock(map); in vm_map_madvise()
3108 rv = vm_map_clip_end(map, entry, end); in vm_map_madvise()
3110 vm_map_unlock(map); in vm_map_madvise()
3142 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3144 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3145 vm_map_unlock(map); in vm_map_madvise()
3156 if (!vm_map_lookup_entry(map, start, &entry)) in vm_map_madvise()
3206 pmap_advise(map->pmap, useStart, useEnd, in vm_map_madvise()
3219 vm_map_pmap_enter(map, in vm_map_madvise()
3229 vm_map_unlock_read(map); in vm_map_madvise()
3243 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_inherit() argument
3260 vm_map_lock(map); in vm_map_inherit()
3261 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
3262 rv = vm_map_lookup_clip_start(map, start, &start_entry, &prev_entry); in vm_map_inherit()
3265 if (vm_map_lookup_entry(map, end - 1, &lentry)) { in vm_map_inherit()
3266 rv = vm_map_clip_end(map, lentry, end); in vm_map_inherit()
3287 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3289 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3291 vm_map_unlock(map); in vm_map_inherit()
3304 vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start, in vm_map_entry_in_transition() argument
3311 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_in_transition()
3319 last_timestamp = map->timestamp; in vm_map_entry_in_transition()
3320 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_entry_in_transition()
3325 vm_map_lock(map); in vm_map_entry_in_transition()
3326 if (last_timestamp + 1 == map->timestamp) in vm_map_entry_in_transition()
3334 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_entry_in_transition()
3350 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_unwire() argument
3361 vm_map_lock(map); in vm_map_unwire()
3362 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire()
3363 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3367 vm_map_unlock(map); in vm_map_unwire()
3377 next_entry = vm_map_entry_in_transition(map, start, in vm_map_unwire()
3381 vm_map_unlock(map); in vm_map_unwire()
3391 rv = vm_map_clip_start(map, entry, start); in vm_map_unwire()
3394 rv = vm_map_clip_end(map, entry, end); in vm_map_unwire()
3430 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3459 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3475 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3477 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3478 vm_map_unlock(map); in vm_map_unwire()
3480 vm_map_wakeup(map); in vm_map_unwire()
3514 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3518 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_entry_failure()
3530 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3543 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire() argument
3547 vm_map_lock(map); in vm_map_wire()
3548 rv = vm_map_wire_locked(map, start, end, flags); in vm_map_wire()
3549 vm_map_unlock(map); in vm_map_wire()
3560 vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire_locked() argument
3570 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_locked()
3579 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire_locked()
3580 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3591 next_entry = vm_map_entry_in_transition(map, start, in vm_map_wire_locked()
3603 rv = vm_map_clip_start(map, entry, start); in vm_map_wire_locked()
3606 rv = vm_map_clip_end(map, entry, end); in vm_map_wire_locked()
3632 vm_map_wire_entry_failure(map, entry, in vm_map_wire_locked()
3645 last_timestamp = map->timestamp; in vm_map_wire_locked()
3648 vm_map_busy(map); in vm_map_wire_locked()
3649 vm_map_unlock(map); in vm_map_wire_locked()
3657 rv = vm_fault(map, faddr, VM_PROT_NONE, in vm_map_wire_locked()
3662 vm_map_lock(map); in vm_map_wire_locked()
3663 vm_map_unbusy(map); in vm_map_wire_locked()
3664 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire_locked()
3671 if (!vm_map_lookup_entry(map, saved_start, in vm_map_wire_locked()
3687 vm_map_wire_entry_failure(map, in vm_map_wire_locked()
3692 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire_locked()
3718 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3766 vm_map_entry_unwire(map, entry); in vm_map_wire_locked()
3784 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3786 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3788 vm_map_wakeup(map); in vm_map_wire_locked()
3810 vm_map_t map, in vm_map_sync() argument
3824 vm_map_lock_read(map); in vm_map_sync()
3825 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_sync()
3826 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_sync()
3827 vm_map_unlock_read(map); in vm_map_sync()
3841 vm_map_unlock_read(map); in vm_map_sync()
3848 vm_map_unlock_read(map); in vm_map_sync()
3855 vm_map_unlock_read(map); in vm_map_sync()
3861 pmap_remove(map->pmap, start, end); in vm_map_sync()
3889 last_timestamp = map->timestamp; in vm_map_sync()
3890 vm_map_unlock_read(map); in vm_map_sync()
3895 vm_map_lock_read(map); in vm_map_sync()
3896 if (last_timestamp == map->timestamp || in vm_map_sync()
3897 !vm_map_lookup_entry(map, start, &entry)) in vm_map_sync()
3901 vm_map_unlock_read(map); in vm_map_sync()
3914 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3918 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unwire()
3925 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3946 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3952 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3959 vm_map_entry_deallocate(entry, vm_map_is_system(map)); in vm_map_entry_delete()
3964 map->size -= size; in vm_map_entry_delete()
4006 if (vm_map_is_system(map)) in vm_map_entry_delete()
4021 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_delete() argument
4026 VM_MAP_ASSERT_LOCKED(map); in vm_map_delete()
4035 rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); in vm_map_delete()
4045 (vm_map_pmap(map) != kernel_pmap && in vm_map_delete()
4052 last_timestamp = map->timestamp; in vm_map_delete()
4053 (void) vm_map_unlock_and_wait(map, 0); in vm_map_delete()
4054 vm_map_lock(map); in vm_map_delete()
4055 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
4062 rv = vm_map_lookup_clip_start(map, saved_start, in vm_map_delete()
4072 rv = vm_map_clip_end(map, entry, end); in vm_map_delete()
4082 vm_map_entry_unwire(map, entry); in vm_map_delete()
4091 pmap_map_delete(map->pmap, entry->start, entry->end); in vm_map_delete()
4099 vm_map_entry_delete(map, entry); in vm_map_delete()
4111 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_remove() argument
4115 vm_map_lock(map); in vm_map_remove()
4116 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_remove()
4117 result = vm_map_delete(map, start, end); in vm_map_remove()
4118 vm_map_unlock(map); in vm_map_remove()
4137 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_check_protection() argument
4143 if (!vm_map_lookup_entry(map, start, &tmp_entry)) in vm_map_check_protection()
4581 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack() argument
4588 MPASS((map->flags & MAP_WIREFUTURE) == 0); in vm_map_stack()
4591 vm_map_lock(map); in vm_map_stack()
4594 if (map->size + init_ssize > vmemlim) { in vm_map_stack()
4598 rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, in vm_map_stack()
4601 vm_map_unlock(map); in vm_map_stack()
4611 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack_locked() argument
4623 !vm_map_range_valid(map, addrbos, addrbos + max_ssize)) in vm_map_stack_locked()
4636 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) in vm_map_stack_locked()
4657 rv = vm_map_insert1(map, NULL, 0, bot, top, prot, max, cow, in vm_map_stack_locked()
4667 rv = vm_map_insert1(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, in vm_map_stack_locked()
4689 (void)vm_map_delete(map, bot, top); in vm_map_stack_locked()
4699 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) in vm_map_growstack() argument
4725 if (p != initproc && (map != &p->p_vmspace->vm_map || in vm_map_growstack()
4729 MPASS(!vm_map_is_system(map)); in vm_map_growstack()
4736 if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) in vm_map_growstack()
4796 if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4797 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { in vm_map_growstack()
4805 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { in vm_map_growstack()
4816 if (map->size + grow_amount > vmemlim) { in vm_map_growstack()
4823 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { in vm_map_growstack()
4832 if (vm_map_lock_upgrade(map)) { in vm_map_growstack()
4834 vm_map_lock_read(map); in vm_map_growstack()
4850 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4854 vm_map_entry_resize(map, gap_entry, -grow_amount); in vm_map_growstack()
4857 rv = vm_map_insert(map, NULL, 0, grow_start, in vm_map_growstack()
4861 rv1 = vm_map_insert1(map, NULL, 0, gap_start, in vm_map_growstack()
4869 vm_map_entry_resize(map, gap_entry, in vm_map_growstack()
4879 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4880 rv = vm_map_wire_locked(map, grow_start, in vm_map_growstack()
4884 vm_map_lock_downgrade(map); in vm_map_growstack()
4890 error = racct_set(p, RACCT_VMEM, map->size); in vm_map_growstack()
4894 ptoa(pmap_wired_count(map->pmap))); in vm_map_growstack()
5012 vm_map_t map = *var_map; in vm_map_lookup() local
5021 vm_map_lock_read(map); in vm_map_lookup()
5027 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { in vm_map_lookup()
5028 vm_map_unlock_read(map); in vm_map_lookup()
5038 vm_map_t old_map = map; in vm_map_lookup()
5040 *var_map = map = entry->object.sub_map; in vm_map_lookup()
5051 if (prot == VM_PROT_NONE && map != kernel_map && in vm_map_lookup()
5054 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
5059 vm_map_unlock_read(map); in vm_map_lookup()
5069 vm_map_unlock_read(map); in vm_map_lookup()
5101 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5113 vm_map_unlock(map); in vm_map_lookup()
5131 vm_map_lock_downgrade(map); in vm_map_lookup()
5144 if (entry->object.vm_object == NULL && !vm_map_is_system(map)) { in vm_map_lookup()
5145 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5151 vm_map_lock_downgrade(map); in vm_map_lookup()
5182 vm_map_t map = *var_map; in vm_map_lookup_locked() local
5189 if (!vm_map_lookup_entry(map, vaddr, out_entry)) in vm_map_lookup_locked()
5232 if (entry->object.vm_object == NULL && !vm_map_is_system(map)) in vm_map_lookup_locked()
5253 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
5258 vm_map_unlock_read(map); in vm_map_lookup_done()
5262 vm_map_max_KBI(const struct vm_map *map) in vm_map_max_KBI() argument
5265 return (vm_map_max(map)); in vm_map_max_KBI()
5269 vm_map_min_KBI(const struct vm_map *map) in vm_map_min_KBI() argument
5272 return (vm_map_min(map)); in vm_map_min_KBI()
5276 vm_map_pmap_KBI(vm_map_t map) in vm_map_pmap_KBI() argument
5279 return (map->pmap); in vm_map_pmap_KBI()
5283 vm_map_range_valid_KBI(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_range_valid_KBI() argument
5286 return (vm_map_range_valid(map, start, end)); in vm_map_range_valid_KBI()
5291 _vm_map_assert_consistent(vm_map_t map, int check) in _vm_map_assert_consistent() argument
5298 ++map->nupdates; in _vm_map_assert_consistent()
5303 header = prev = &map->header; in _vm_map_assert_consistent()
5304 VM_MAP_ENTRY_FOREACH(entry, map) { in _vm_map_assert_consistent()
5306 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5309 ("map %p start = %jx, end = %jx", map, in _vm_map_assert_consistent()
5313 ("map %p left->start = %jx, start = %jx", map, in _vm_map_assert_consistent()
5317 ("map %p start = %jx, right->start = %jx", map, in _vm_map_assert_consistent()
5319 cur = map->root; in _vm_map_assert_consistent()
5327 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5333 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5337 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5344 ("map %p max = %jx, max_left = %jx, max_right = %jx", map, in _vm_map_assert_consistent()
5350 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5362 vm_map_print(vm_map_t map) in vm_map_print() argument
5367 (void *)map, in vm_map_print()
5368 (void *)map->pmap, map->nentries, map->timestamp); in vm_map_print()
5371 prev = &map->header; in vm_map_print()
5372 VM_MAP_ENTRY_FOREACH(entry, map) { in vm_map_print()
5392 if (prev == &map->header || in vm_map_print()
5413 if (prev == &map->header || in vm_map_print()
5428 DB_SHOW_COMMAND(map, map) in DB_SHOW_COMMAND() argument