Lines Matching full:entry
114 * [That is, an entry is split into two, bordering at a
131 static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map);
132 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
133 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
144 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
251 * User map and entry structures are allocated from the general purpose
262 * Disable the use of per-CPU buckets: map entry allocation is in vm_map_startup()
265 kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry), in vm_map_startup()
269 /* Reserve an extra map entry for use when replenishing the reserve. */ in vm_map_startup()
276 mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry), in vm_map_startup()
535 vm_map_entry_set_vnode_text(vm_map_entry_t entry, bool add) in vm_map_entry_set_vnode_text() argument
541 if ((entry->eflags & MAP_ENTRY_VN_EXEC) == 0) in vm_map_entry_set_vnode_text()
543 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, in vm_map_entry_set_vnode_text()
545 object = entry->object.vm_object; in vm_map_entry_set_vnode_text()
546 KASSERT(object != NULL, ("No object for text, entry %p", entry)); in vm_map_entry_set_vnode_text()
552 KASSERT(object != NULL, ("No content object for text, entry %p obj %p", in vm_map_entry_set_vnode_text()
553 entry, entry->object.vm_object)); in vm_map_entry_set_vnode_text()
557 * referenced by the entry we are processing, so it cannot go in vm_map_entry_set_vnode_text()
584 vm_map_entry_t entry, next; in vm_map_process_deferred() local
588 entry = td->td_map_def_user; in vm_map_process_deferred()
590 while (entry != NULL) { in vm_map_process_deferred()
591 next = entry->defer_next; in vm_map_process_deferred()
592 MPASS((entry->eflags & (MAP_ENTRY_WRITECNT | in vm_map_process_deferred()
595 if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) { in vm_map_process_deferred()
600 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, in vm_map_process_deferred()
602 object = entry->object.vm_object; in vm_map_process_deferred()
604 vm_pager_release_writecount(object, entry->start, in vm_map_process_deferred()
605 entry->end); in vm_map_process_deferred()
607 vm_map_entry_set_vnode_text(entry, false); in vm_map_process_deferred()
608 vm_map_entry_deallocate(entry, FALSE); in vm_map_process_deferred()
609 entry = next; in vm_map_process_deferred()
926 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
928 uma_zfree(vm_map_is_system(map) ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
934 * Allocates a VM map entry for insertion.
935 * No entry fields are filled in.
950 * map entry, dipping into the reserve if necessary, and set a in vm_map_entry_create()
979 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior) in vm_map_entry_set_behavior() argument
981 entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) | in vm_map_entry_set_behavior()
1011 * Find the {predecessor, successor} of the entry by taking one step
1016 vm_map_entry_pred(vm_map_entry_t entry) in vm_map_entry_pred() argument
1020 prior = entry->left; in vm_map_entry_pred()
1021 if (prior->right->start < entry->start) { in vm_map_entry_pred()
1024 while (prior->right != entry); in vm_map_entry_pred()
1327 * is the predecessor of the first map entry, and the successor of the
1331 * adjacent entry (lower if possible) if addr is not in the tree.
1379 * Insert/remove entries from maps. On linking, if new entry clips
1380 * existing entry, trim existing entry to avoid overlap, and manage
1381 * offsets. On unlinking, merge disappearing entry with neighbor, if
1386 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1392 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1393 map->nentries, entry); in vm_map_entry_link()
1397 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1400 * The new entry does not overlap any existing entry in the in vm_map_entry_link()
1403 max_free_left = vm_map_splay_merge_pred(header, entry, llist); in vm_map_entry_link()
1404 max_free_right = vm_map_splay_merge_succ(header, entry, rlist); in vm_map_entry_link()
1405 } else if (entry->start == root->start) { in vm_map_entry_link()
1407 * The new entry is a clone of root, with only the end field in vm_map_entry_link()
1408 * changed. The root entry will be shrunk to abut the new in vm_map_entry_link()
1409 * entry, and will be the right child of the new root entry in in vm_map_entry_link()
1412 KASSERT(entry->end < root->end, in vm_map_entry_link()
1413 ("%s: clip_start not within entry", __func__)); in vm_map_entry_link()
1416 root->offset += entry->end - root->start; in vm_map_entry_link()
1417 root->start = entry->end; in vm_map_entry_link()
1418 max_free_left = vm_map_splay_merge_pred(header, entry, llist); in vm_map_entry_link()
1420 vm_map_splay_merge_pred(entry, root, entry), in vm_map_entry_link()
1424 * The new entry is a clone of root, with only the start field in vm_map_entry_link()
1425 * changed. The root entry will be shrunk to abut the new in vm_map_entry_link()
1426 * entry, and will be the left child of the new root entry in in vm_map_entry_link()
1429 KASSERT(entry->end == root->end, in vm_map_entry_link()
1430 ("%s: clip_start not within entry", __func__)); in vm_map_entry_link()
1432 if ((entry->eflags & MAP_ENTRY_STACK_GAP) == 0) in vm_map_entry_link()
1433 entry->offset += entry->start - root->start; in vm_map_entry_link()
1434 root->end = entry->start; in vm_map_entry_link()
1437 vm_map_splay_merge_succ(entry, root, entry)); in vm_map_entry_link()
1438 max_free_right = vm_map_splay_merge_succ(header, entry, rlist); in vm_map_entry_link()
1440 entry->max_free = vm_size_max(max_free_left, max_free_right); in vm_map_entry_link()
1441 map->root = entry; in vm_map_entry_link()
1451 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1459 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1489 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1490 map->nentries, entry); in vm_map_entry_unlink()
1502 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) in vm_map_entry_resize() argument
1508 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize()
1511 entry->end += grow_amount; in vm_map_entry_resize()
1517 CTR4(KTR_VM, "%s: map %p, nentries %d, entry %p", in vm_map_entry_resize()
1518 __func__, map, map->nentries, entry); in vm_map_entry_resize()
1524 * Finds the map entry containing (or
1526 * in the given map; the entry is returned
1527 * in the "entry" parameter. The boolean
1535 vm_map_entry_t *entry) /* OUT */ in vm_map_lookup_entry() argument
1541 * If the map is empty, then the map entry immediately preceding in vm_map_lookup_entry()
1547 *entry = header; in vm_map_lookup_entry()
1551 *entry = cur; in vm_map_lookup_entry()
1569 * If "address" is contained within a map entry, the new root in vm_map_lookup_entry()
1570 * is that map entry. Otherwise, the new root is a map entry in vm_map_lookup_entry()
1574 *entry = header; in vm_map_lookup_entry()
1577 *entry = cur; in vm_map_lookup_entry()
1597 *entry = cur; in vm_map_lookup_entry()
1601 *entry = lbound; in vm_map_lookup_entry()
1607 * returns the newly inserted map entry in '*res'. In case the new
1608 * entry is coalesced with a neighbor or an existing entry was
1609 * resized, that entry is returned. In any case, the returned entry
1647 * Find the entry prior to the proposed starting address; if it's part in vm_map_insert1()
1648 * of an existing entry, this range is bogus. in vm_map_insert1()
1654 * Assert that the next entry doesn't overlap the end point. in vm_map_insert1()
1764 * can extend the previous map entry to include the in vm_map_insert1()
1785 * map entry, we have to create a new map entry. We in vm_map_insert1()
1803 * Create a new entry in vm_map_insert1()
1827 * Insert the new entry into the list in vm_map_insert1()
1834 * Try to coalesce the new entry with both the previous and next in vm_map_insert1()
1836 * with the previous entry when object is NULL. Here, we handle the in vm_map_insert1()
1859 * prior to making call to account for the new entry.
1878 * contiguous free space between an entry in its subtree and a
1879 * neighbor of that entry. This allows finding a free region in
1910 * After splay_split, if start is within an entry, push it to the start in vm_map_findspace()
2130 * prior to making call to account for the new entry.
2343 * A map entry with any of the following flags set must not be merged with
2344 * another entry.
2351 vm_map_mergeable_neighbors(vm_map_entry_t prev, vm_map_entry_t entry) in vm_map_mergeable_neighbors() argument
2355 (entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0, in vm_map_mergeable_neighbors()
2357 prev, entry)); in vm_map_mergeable_neighbors()
2358 return (prev->end == entry->start && in vm_map_mergeable_neighbors()
2359 prev->object.vm_object == entry->object.vm_object && in vm_map_mergeable_neighbors()
2361 prev->offset + (prev->end - prev->start) == entry->offset) && in vm_map_mergeable_neighbors()
2362 prev->eflags == entry->eflags && in vm_map_mergeable_neighbors()
2363 prev->protection == entry->protection && in vm_map_mergeable_neighbors()
2364 prev->max_protection == entry->max_protection && in vm_map_mergeable_neighbors()
2365 prev->inheritance == entry->inheritance && in vm_map_mergeable_neighbors()
2366 prev->wired_count == entry->wired_count && in vm_map_mergeable_neighbors()
2367 prev->cred == entry->cred); in vm_map_mergeable_neighbors()
2371 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2382 * should not be adjusted when the entry is disposed of. in vm_map_merged_neighbor_dispose()
2384 if (entry->object.vm_object != NULL) in vm_map_merged_neighbor_dispose()
2385 vm_object_deallocate(entry->object.vm_object); in vm_map_merged_neighbor_dispose()
2386 if (entry->cred != NULL) in vm_map_merged_neighbor_dispose()
2387 crfree(entry->cred); in vm_map_merged_neighbor_dispose()
2388 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2397 * the map entry that includes the first range.
2403 vm_map_entry_t entry) in vm_map_try_merge_entries() argument
2407 if ((entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 && in vm_map_try_merge_entries()
2408 vm_map_mergeable_neighbors(prev_entry, entry)) { in vm_map_try_merge_entries()
2411 return (entry); in vm_map_try_merge_entries()
2419 * Allocate an object to back a map entry.
2422 vm_map_entry_back(vm_map_entry_t entry) in vm_map_entry_back() argument
2426 KASSERT(entry->object.vm_object == NULL, in vm_map_entry_back()
2427 ("map entry %p has backing object", entry)); in vm_map_entry_back()
2428 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, in vm_map_entry_back()
2429 ("map entry %p is a submap", entry)); in vm_map_entry_back()
2430 object = vm_object_allocate_anon(atop(entry->end - entry->start), NULL, in vm_map_entry_back()
2431 entry->cred); in vm_map_entry_back()
2432 entry->object.vm_object = object; in vm_map_entry_back()
2433 entry->offset = 0; in vm_map_entry_back()
2434 entry->cred = NULL; in vm_map_entry_back()
2440 * If there is no object backing this entry, create one. Otherwise, if
2441 * the entry has cred, give it to the backing object.
2444 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_charge_object() argument
2449 KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, in vm_map_entry_charge_object()
2450 ("map entry %p is a submap", entry)); in vm_map_entry_charge_object()
2451 object = entry->object.vm_object; in vm_map_entry_charge_object()
2453 (entry->eflags & MAP_ENTRY_GUARD) == 0) in vm_map_entry_charge_object()
2454 vm_map_entry_back(entry); in vm_map_entry_charge_object()
2456 ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) && in vm_map_entry_charge_object()
2457 entry->cred != NULL) { in vm_map_entry_charge_object()
2460 ("OVERCOMMIT: %s: both cred e %p", __func__, entry)); in vm_map_entry_charge_object()
2461 object->cred = entry->cred; in vm_map_entry_charge_object()
2462 if (entry->end - entry->start < ptoa(object->size)) { in vm_map_entry_charge_object()
2464 entry->end + entry->start, object->cred); in vm_map_entry_charge_object()
2466 VM_OBJECT_WUNLOCK(entry->object.vm_object); in vm_map_entry_charge_object()
2467 entry->cred = NULL; in vm_map_entry_charge_object()
2474 * Create a duplicate map entry for clipping.
2477 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_clone() argument
2485 * objects won't be created after the map entry is split. in vm_map_entry_clone()
2487 vm_map_entry_charge_object(map, entry); in vm_map_entry_clone()
2489 /* Clone the entry. */ in vm_map_entry_clone()
2491 *new_entry = *entry; in vm_map_entry_clone()
2493 crhold(entry->cred); in vm_map_entry_clone()
2494 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { in vm_map_entry_clone()
2499 * MAP_ENTRY_WRITECNT type entry shall be kept as is here. The in vm_map_entry_clone()
2510 * Asserts that the given entry begins at or after
2512 * it splits the entry into two.
2515 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) in vm_map_clip_start() argument
2522 "%s: map %p entry %p start 0x%jx", __func__, map, entry, in vm_map_clip_start()
2525 if (startaddr <= entry->start) in vm_map_clip_start()
2529 KASSERT(entry->end > startaddr && entry->start < startaddr, in vm_map_clip_start()
2530 ("%s: invalid clip of entry %p", __func__, entry)); in vm_map_clip_start()
2532 bdry_idx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(entry); in vm_map_clip_start()
2538 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_start()
2541 * Split off the front portion. Insert the new entry BEFORE this one, in vm_map_clip_start()
2542 * so that this entry has the specified starting address. in vm_map_clip_start()
2552 * Find the entry at or just after 'start', and clip it if 'start' is in
2553 * the interior of the entry. Return entry after 'start', and in
2554 * prev_entry set the entry before 'start'.
2560 vm_map_entry_t entry; in vm_map_lookup_clip_start() local
2569 entry = *prev_entry; in vm_map_lookup_clip_start()
2570 rv = vm_map_clip_start(map, entry, start); in vm_map_lookup_clip_start()
2573 *prev_entry = vm_map_entry_pred(entry); in vm_map_lookup_clip_start()
2575 entry = vm_map_entry_succ(*prev_entry); in vm_map_lookup_clip_start()
2576 *res_entry = entry; in vm_map_lookup_clip_start()
2583 * Asserts that the given entry ends at or before
2585 * it splits the entry into two.
2588 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) in vm_map_clip_end() argument
2595 "%s: map %p entry %p end 0x%jx", __func__, map, entry, in vm_map_clip_end()
2598 if (endaddr >= entry->end) in vm_map_clip_end()
2602 KASSERT(entry->start < endaddr && entry->end > endaddr, in vm_map_clip_end()
2603 ("%s: invalid clip of entry %p", __func__, entry)); in vm_map_clip_end()
2605 bdry_idx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(entry); in vm_map_clip_end()
2611 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_end()
2614 * Split off the back portion. Insert the new entry AFTER this one, in vm_map_clip_end()
2615 * so that this entry has the specified ending address. in vm_map_clip_end()
2648 vm_map_entry_t entry; in vm_map_submap() local
2659 if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && in vm_map_submap()
2660 (entry->eflags & MAP_ENTRY_COW) == 0 && in vm_map_submap()
2661 entry->object.vm_object == NULL) { in vm_map_submap()
2662 result = vm_map_clip_start(map, entry, start); in vm_map_submap()
2665 result = vm_map_clip_end(map, entry, end); in vm_map_submap()
2668 entry->object.sub_map = submap; in vm_map_submap()
2669 entry->eflags |= MAP_ENTRY_IS_SUB_MAP; in vm_map_submap()
2785 vm_map_protect_guard(vm_map_entry_t entry, vm_prot_t new_prot, in vm_map_protect_guard() argument
2790 MPASS((entry->eflags & MAP_ENTRY_GUARD) != 0); in vm_map_protect_guard()
2791 if ((entry->eflags & MAP_ENTRY_STACK_GAP) == 0) in vm_map_protect_guard()
2794 old_prot = PROT_EXTRACT(entry->offset); in vm_map_protect_guard()
2796 entry->offset = PROT_MAX(new_maxprot) | in vm_map_protect_guard()
2800 entry->offset = new_prot | PROT_MAX( in vm_map_protect_guard()
2801 PROT_MAX_EXTRACT(entry->offset)); in vm_map_protect_guard()
2815 vm_map_entry_t entry, first_entry, in_tran, prev_entry; in vm_map_protect() local
2847 * update the protection on the map entry in between faults. in vm_map_protect()
2880 for (entry = first_entry; entry->start < end; in vm_map_protect()
2881 entry = vm_map_entry_succ(entry)) { in vm_map_protect()
2882 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) { in vm_map_protect()
2886 if ((entry->eflags & (MAP_ENTRY_GUARD | in vm_map_protect()
2889 max_prot = (entry->eflags & MAP_ENTRY_STACK_GAP) != 0 ? in vm_map_protect()
2890 PROT_MAX_EXTRACT(entry->offset) : entry->max_protection; in vm_map_protect()
2895 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0) in vm_map_protect()
2896 in_tran = entry; in vm_map_protect()
2901 * stabilized. An in-transition entry might already have its pages in vm_map_protect()
2924 for (entry = first_entry; entry->start < end; in vm_map_protect()
2925 entry = vm_map_entry_succ(entry)) { in vm_map_protect()
2926 rv = vm_map_clip_end(map, entry, end); in vm_map_protect()
2933 ((new_prot & ~entry->protection) & VM_PROT_WRITE) == 0 || in vm_map_protect()
2934 ENTRY_CHARGED(entry) || in vm_map_protect()
2935 (entry->eflags & MAP_ENTRY_GUARD) != 0) in vm_map_protect()
2939 obj = entry->object.vm_object; in vm_map_protect()
2942 (entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0) { in vm_map_protect()
2943 if (!swap_reserve(entry->end - entry->start)) { in vm_map_protect()
2945 end = entry->end; in vm_map_protect()
2949 entry->cred = cred; in vm_map_protect()
2965 ("vm_map_protect: object %p overcharged (entry %p)", in vm_map_protect()
2966 obj, entry)); in vm_map_protect()
2970 end = entry->end; in vm_map_protect()
2984 for (prev_entry = vm_map_entry_pred(first_entry), entry = first_entry; in vm_map_protect()
2985 entry->start < end; in vm_map_protect()
2986 vm_map_try_merge_entries(map, prev_entry, entry), in vm_map_protect()
2987 prev_entry = entry, entry = vm_map_entry_succ(entry)) { in vm_map_protect()
2991 if ((entry->eflags & MAP_ENTRY_GUARD) != 0) { in vm_map_protect()
2992 vm_map_protect_guard(entry, new_prot, new_maxprot, in vm_map_protect()
2997 old_prot = entry->protection; in vm_map_protect()
3000 entry->max_protection = new_maxprot; in vm_map_protect()
3001 entry->protection = new_maxprot & old_prot; in vm_map_protect()
3004 entry->protection = new_prot; in vm_map_protect()
3012 if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0 && in vm_map_protect()
3013 (entry->protection & VM_PROT_WRITE) != 0 && in vm_map_protect()
3015 vm_fault_copy_entry(map, map, entry, entry, NULL); in vm_map_protect()
3021 if ((old_prot & ~entry->protection) != 0) { in vm_map_protect()
3022 #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \ in vm_map_protect() argument
3024 pmap_protect(map->pmap, entry->start, in vm_map_protect()
3025 entry->end, in vm_map_protect()
3026 entry->protection & MASK(entry)); in vm_map_protect()
3030 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_protect()
3050 vm_map_entry_t entry, prev_entry; in vm_map_madvise() local
3086 * Locate starting entry and clip if necessary. in vm_map_madvise()
3097 rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); in vm_map_madvise()
3103 for (; entry->start < end; prev_entry = entry, in vm_map_madvise()
3104 entry = vm_map_entry_succ(entry)) { in vm_map_madvise()
3105 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) in vm_map_madvise()
3108 rv = vm_map_clip_end(map, entry, end); in vm_map_madvise()
3116 vm_map_entry_set_behavior(entry, in vm_map_madvise()
3120 vm_map_entry_set_behavior(entry, in vm_map_madvise()
3124 vm_map_entry_set_behavior(entry, in vm_map_madvise()
3128 entry->eflags |= MAP_ENTRY_NOSYNC; in vm_map_madvise()
3131 entry->eflags &= ~MAP_ENTRY_NOSYNC; in vm_map_madvise()
3134 entry->eflags |= MAP_ENTRY_NOCOREDUMP; in vm_map_madvise()
3137 entry->eflags &= ~MAP_ENTRY_NOCOREDUMP; in vm_map_madvise()
3142 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3144 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3156 if (!vm_map_lookup_entry(map, start, &entry)) in vm_map_madvise()
3157 entry = vm_map_entry_succ(entry); in vm_map_madvise()
3158 for (; entry->start < end; in vm_map_madvise()
3159 entry = vm_map_entry_succ(entry)) { in vm_map_madvise()
3162 if ((entry->eflags & (MAP_ENTRY_IS_SUB_MAP | in vm_map_madvise()
3170 * entry's object nor the presence of a in vm_map_madvise()
3174 entry->object.vm_object != NULL && in vm_map_madvise()
3175 entry->object.vm_object->backing_object != NULL) in vm_map_madvise()
3178 pstart = OFF_TO_IDX(entry->offset); in vm_map_madvise()
3179 pend = pstart + atop(entry->end - entry->start); in vm_map_madvise()
3180 useStart = entry->start; in vm_map_madvise()
3181 useEnd = entry->end; in vm_map_madvise()
3183 if (entry->start < start) { in vm_map_madvise()
3184 pstart += atop(start - entry->start); in vm_map_madvise()
3187 if (entry->end > end) { in vm_map_madvise()
3188 pend -= atop(entry->end - end); in vm_map_madvise()
3209 vm_object_madvise(entry->object.vm_object, pstart, in vm_map_madvise()
3218 entry->wired_count == 0) { in vm_map_madvise()
3221 entry->protection, in vm_map_madvise()
3222 entry->object.vm_object, in vm_map_madvise()
3246 vm_map_entry_t entry, lentry, prev_entry, start_entry; in vm_map_inherit() local
3271 for (entry = start_entry; entry->start < end; in vm_map_inherit()
3272 prev_entry = entry, entry = vm_map_entry_succ(entry)) { in vm_map_inherit()
3273 if ((entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) in vm_map_inherit()
3280 for (entry = start_entry; entry->start < end; prev_entry = entry, in vm_map_inherit()
3281 entry = vm_map_entry_succ(entry)) { in vm_map_inherit()
3282 KASSERT(entry->end <= end, ("non-clipped entry %p end %jx %jx", in vm_map_inherit()
3283 entry, (uintmax_t)entry->end, (uintmax_t)end)); in vm_map_inherit()
3284 if ((entry->eflags & MAP_ENTRY_GUARD) == 0 || in vm_map_inherit()
3286 entry->inheritance = new_inheritance; in vm_map_inherit()
3287 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3289 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3298 * Release the map lock, and sleep until the entry is no longer in
3300 * another held the lock, lookup a possibly-changed entry at or after the
3301 * 'start' position of the old entry.
3307 vm_map_entry_t entry; in vm_map_entry_in_transition() local
3313 ("not in-tranition map entry %p", in_entry)); in vm_map_entry_in_transition()
3315 * We have not yet clipped the entry. in vm_map_entry_in_transition()
3330 * Look again for the entry because the map was modified while it was in vm_map_entry_in_transition()
3331 * unlocked. Specifically, the entry may have been clipped, merged, or in vm_map_entry_in_transition()
3334 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_entry_in_transition()
3339 entry = vm_map_entry_succ(entry); in vm_map_entry_in_transition()
3341 return (entry); in vm_map_entry_in_transition()
3353 vm_map_entry_t entry, first_entry, next_entry, prev_entry; in vm_map_unwire() local
3372 for (entry = first_entry; entry->start < end; entry = next_entry) { in vm_map_unwire()
3373 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { in vm_map_unwire()
3375 * We have not yet clipped the entry. in vm_map_unwire()
3378 &end, holes_ok, entry); in vm_map_unwire()
3380 if (entry == first_entry) { in vm_map_unwire()
3387 first_entry = (entry == first_entry) ? in vm_map_unwire()
3391 rv = vm_map_clip_start(map, entry, start); in vm_map_unwire()
3394 rv = vm_map_clip_end(map, entry, end); in vm_map_unwire()
3399 * Mark the entry in case the map lock is released. (See in vm_map_unwire()
3402 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && in vm_map_unwire()
3403 entry->wiring_thread == NULL, in vm_map_unwire()
3404 ("owned map entry %p", entry)); in vm_map_unwire()
3405 entry->eflags |= MAP_ENTRY_IN_TRANSITION; in vm_map_unwire()
3406 entry->wiring_thread = curthread; in vm_map_unwire()
3407 next_entry = vm_map_entry_succ(entry); in vm_map_unwire()
3413 entry->end < end && next_entry->start > entry->end) { in vm_map_unwire()
3414 end = entry->end; in vm_map_unwire()
3419 * If system unwiring, require that the entry is system wired. in vm_map_unwire()
3422 vm_map_entry_system_wired_count(entry) == 0) { in vm_map_unwire()
3423 end = entry->end; in vm_map_unwire()
3433 entry = vm_map_entry_succ(first_entry); in vm_map_unwire()
3436 entry = first_entry; in vm_map_unwire()
3438 for (; entry->start < end; in vm_map_unwire()
3439 prev_entry = entry, entry = vm_map_entry_succ(entry)) { in vm_map_unwire()
3446 * entry. Detect these cases and skip any entries in vm_map_unwire()
3449 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || in vm_map_unwire()
3450 entry->wiring_thread != curthread) { in vm_map_unwire()
3452 ("vm_map_unwire: !HOLESOK and new/changed entry")); in vm_map_unwire()
3457 (entry->eflags & MAP_ENTRY_USER_WIRED))) { in vm_map_unwire()
3458 if (entry->wired_count == 1) in vm_map_unwire()
3459 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3461 entry->wired_count--; in vm_map_unwire()
3463 entry->eflags &= ~MAP_ENTRY_USER_WIRED; in vm_map_unwire()
3465 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, in vm_map_unwire()
3466 ("vm_map_unwire: in-transition flag missing %p", entry)); in vm_map_unwire()
3467 KASSERT(entry->wiring_thread == curthread, in vm_map_unwire()
3468 ("vm_map_unwire: alien wire %p", entry)); in vm_map_unwire()
3469 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; in vm_map_unwire()
3470 entry->wiring_thread = NULL; in vm_map_unwire()
3471 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { in vm_map_unwire()
3472 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; in vm_map_unwire()
3475 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3477 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3509 * Handle a wiring failure on the given entry.
3514 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3519 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 && in vm_map_wire_entry_failure()
3520 entry->wired_count == 1, in vm_map_wire_entry_failure()
3521 ("vm_map_wire_entry_failure: entry %p isn't being wired", entry)); in vm_map_wire_entry_failure()
3522 KASSERT(failed_addr < entry->end, in vm_map_wire_entry_failure()
3523 ("vm_map_wire_entry_failure: entry %p was fully wired", entry)); in vm_map_wire_entry_failure()
3526 * If any pages at the start of this entry were successfully wired, in vm_map_wire_entry_failure()
3529 if (failed_addr > entry->start) { in vm_map_wire_entry_failure()
3530 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3531 vm_object_unwire(entry->object.vm_object, entry->offset, in vm_map_wire_entry_failure()
3532 failed_addr - entry->start, PQ_ACTIVE); in vm_map_wire_entry_failure()
3537 * entry. in vm_map_wire_entry_failure()
3539 entry->wired_count = -1; in vm_map_wire_entry_failure()
3562 vm_map_entry_t entry, first_entry, next_entry, prev_entry; in vm_map_wire_locked() local
3586 for (entry = first_entry; entry->start < end; entry = next_entry) { in vm_map_wire_locked()
3587 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { in vm_map_wire_locked()
3589 * We have not yet clipped the entry. in vm_map_wire_locked()
3592 &end, holes_ok, entry); in vm_map_wire_locked()
3594 if (entry == first_entry) in vm_map_wire_locked()
3599 first_entry = (entry == first_entry) ? in vm_map_wire_locked()
3603 rv = vm_map_clip_start(map, entry, start); in vm_map_wire_locked()
3606 rv = vm_map_clip_end(map, entry, end); in vm_map_wire_locked()
3611 * Mark the entry in case the map lock is released. (See in vm_map_wire_locked()
3614 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && in vm_map_wire_locked()
3615 entry->wiring_thread == NULL, in vm_map_wire_locked()
3616 ("owned map entry %p", entry)); in vm_map_wire_locked()
3617 entry->eflags |= MAP_ENTRY_IN_TRANSITION; in vm_map_wire_locked()
3618 entry->wiring_thread = curthread; in vm_map_wire_locked()
3619 if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 in vm_map_wire_locked()
3620 || (entry->protection & prot) != prot) { in vm_map_wire_locked()
3621 entry->eflags |= MAP_ENTRY_WIRE_SKIPPED; in vm_map_wire_locked()
3623 end = entry->end; in vm_map_wire_locked()
3627 } else if (entry->wired_count == 0) { in vm_map_wire_locked()
3628 entry->wired_count++; in vm_map_wire_locked()
3630 npages = atop(entry->end - entry->start); in vm_map_wire_locked()
3632 vm_map_wire_entry_failure(map, entry, in vm_map_wire_locked()
3633 entry->start); in vm_map_wire_locked()
3634 end = entry->end; in vm_map_wire_locked()
3643 saved_start = entry->start; in vm_map_wire_locked()
3644 saved_end = entry->end; in vm_map_wire_locked()
3646 bidx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(entry); in vm_map_wire_locked()
3666 * Look again for the entry because the map was in vm_map_wire_locked()
3667 * modified while it was unlocked. The entry in vm_map_wire_locked()
3675 first_entry = (entry == first_entry) ? in vm_map_wire_locked()
3677 for (entry = next_entry; entry->end < saved_end; in vm_map_wire_locked()
3678 entry = vm_map_entry_succ(entry)) { in vm_map_wire_locked()
3686 faddr < entry->end) in vm_map_wire_locked()
3688 entry, faddr); in vm_map_wire_locked()
3692 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire_locked()
3695 end = entry->end; in vm_map_wire_locked()
3699 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { in vm_map_wire_locked()
3700 entry->wired_count++; in vm_map_wire_locked()
3706 next_entry = vm_map_entry_succ(entry); in vm_map_wire_locked()
3708 entry->end < end && next_entry->start > entry->end) { in vm_map_wire_locked()
3709 end = entry->end; in vm_map_wire_locked()
3721 entry = vm_map_entry_succ(first_entry); in vm_map_wire_locked()
3724 entry = first_entry; in vm_map_wire_locked()
3726 for (; entry->start < end; in vm_map_wire_locked()
3727 prev_entry = entry, entry = vm_map_entry_succ(entry)) { in vm_map_wire_locked()
3734 * wiring this new mapping entry. Detect these cases in vm_map_wire_locked()
3737 * Another way to get an entry not marked with in vm_map_wire_locked()
3741 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || in vm_map_wire_locked()
3742 entry->wiring_thread != curthread) { in vm_map_wire_locked()
3744 ("vm_map_wire: !HOLESOK and new/changed entry")); in vm_map_wire_locked()
3748 if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) { in vm_map_wire_locked()
3752 entry->eflags |= MAP_ENTRY_USER_WIRED; in vm_map_wire_locked()
3753 } else if (entry->wired_count == -1) { in vm_map_wire_locked()
3755 * Wiring failed on this entry. Thus, unwiring is in vm_map_wire_locked()
3758 entry->wired_count = 0; in vm_map_wire_locked()
3760 (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { in vm_map_wire_locked()
3762 * Undo the wiring. Wiring succeeded on this entry in vm_map_wire_locked()
3763 * but failed on a later entry. in vm_map_wire_locked()
3765 if (entry->wired_count == 1) { in vm_map_wire_locked()
3766 vm_map_entry_unwire(map, entry); in vm_map_wire_locked()
3769 atop(entry->end - entry->start)); in vm_map_wire_locked()
3771 entry->wired_count--; in vm_map_wire_locked()
3773 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, in vm_map_wire_locked()
3774 ("vm_map_wire: in-transition flag missing %p", entry)); in vm_map_wire_locked()
3775 KASSERT(entry->wiring_thread == curthread, in vm_map_wire_locked()
3776 ("vm_map_wire: alien wire %p", entry)); in vm_map_wire_locked()
3777 entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION | in vm_map_wire_locked()
3779 entry->wiring_thread = NULL; in vm_map_wire_locked()
3780 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { in vm_map_wire_locked()
3781 entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; in vm_map_wire_locked()
3784 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3786 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3816 vm_map_entry_t entry, first_entry, next_entry; in vm_map_sync() local
3838 for (entry = first_entry; entry->start < end; entry = next_entry) { in vm_map_sync()
3840 if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) { in vm_map_sync()
3844 bdry_idx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(entry); in vm_map_sync()
3852 next_entry = vm_map_entry_succ(entry); in vm_map_sync()
3853 if (end > entry->end && in vm_map_sync()
3854 entry->end != next_entry->start) { in vm_map_sync()
3868 for (entry = first_entry; entry->start < end;) { in vm_map_sync()
3869 offset = entry->offset + (start - entry->start); in vm_map_sync()
3870 size = (end <= entry->end ? end : entry->end) - start; in vm_map_sync()
3871 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) { in vm_map_sync()
3876 smap = entry->object.sub_map; in vm_map_sync()
3886 object = entry->object.vm_object; in vm_map_sync()
3897 !vm_map_lookup_entry(map, start, &entry)) in vm_map_sync()
3898 entry = vm_map_entry_succ(entry); in vm_map_sync()
3908 * Make the region specified by this entry pageable.
3914 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3919 KASSERT(entry->wired_count > 0, in vm_map_entry_unwire()
3920 ("vm_map_entry_unwire: entry %p isn't wired", entry)); in vm_map_entry_unwire()
3922 size = entry->end - entry->start; in vm_map_entry_unwire()
3923 if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) in vm_map_entry_unwire()
3925 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3926 vm_object_unwire(entry->object.vm_object, entry->offset, size, in vm_map_entry_unwire()
3928 entry->wired_count = 0; in vm_map_entry_unwire()
3932 vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) in vm_map_entry_deallocate() argument
3935 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) in vm_map_entry_deallocate()
3936 vm_object_deallocate(entry->object.vm_object); in vm_map_entry_deallocate()
3937 uma_zfree(system_map ? kmapentzone : mapentzone, entry); in vm_map_entry_deallocate()
3943 * Deallocate the given entry from the target map.
3946 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3952 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3953 object = entry->object.vm_object; in vm_map_entry_delete()
3955 if ((entry->eflags & MAP_ENTRY_GUARD) != 0) { in vm_map_entry_delete()
3956 MPASS(entry->cred == NULL); in vm_map_entry_delete()
3957 MPASS((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0); in vm_map_entry_delete()
3959 vm_map_entry_deallocate(entry, vm_map_is_system(map)); in vm_map_entry_delete()
3963 size = entry->end - entry->start; in vm_map_entry_delete()
3966 if (entry->cred != NULL) { in vm_map_entry_delete()
3967 swap_release_by_cred(size, entry->cred); in vm_map_entry_delete()
3968 crfree(entry->cred); in vm_map_entry_delete()
3971 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || object == NULL) { in vm_map_entry_delete()
3972 entry->object.vm_object = NULL; in vm_map_entry_delete()
3975 KASSERT(entry->cred == NULL || object->cred == NULL || in vm_map_entry_delete()
3976 (entry->eflags & MAP_ENTRY_NEEDS_COPY), in vm_map_entry_delete()
3977 ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry)); in vm_map_entry_delete()
3978 offidxstart = OFF_TO_IDX(entry->offset); in vm_map_entry_delete()
4007 vm_map_entry_deallocate(entry, TRUE); in vm_map_entry_delete()
4009 entry->defer_next = curthread->td_map_def_user; in vm_map_entry_delete()
4010 curthread->td_map_def_user = entry; in vm_map_entry_delete()
4023 vm_map_entry_t entry, next_entry, scratch_entry; in vm_map_delete() local
4035 rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); in vm_map_delete()
4038 for (; entry->start < end; entry = next_entry) { in vm_map_delete()
4040 * Wait for wiring or unwiring of an entry to complete. in vm_map_delete()
4044 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || in vm_map_delete()
4046 vm_map_entry_system_wired_count(entry) != 0)) { in vm_map_delete()
4050 saved_start = entry->start; in vm_map_delete()
4051 entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; in vm_map_delete()
4057 * Look again for the entry because the map was in vm_map_delete()
4059 * Specifically, the entry may have been in vm_map_delete()
4067 next_entry = entry; in vm_map_delete()
4072 rv = vm_map_clip_end(map, entry, end); in vm_map_delete()
4075 next_entry = vm_map_entry_succ(entry); in vm_map_delete()
4081 if (entry->wired_count != 0) in vm_map_delete()
4082 vm_map_entry_unwire(map, entry); in vm_map_delete()
4089 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || in vm_map_delete()
4090 entry->object.vm_object != NULL) in vm_map_delete()
4091 pmap_map_delete(map->pmap, entry->start, entry->end); in vm_map_delete()
4094 * Delete the entry only after removing all pmap in vm_map_delete()
4099 vm_map_entry_delete(map, entry); in vm_map_delete()
4140 vm_map_entry_t entry; in vm_map_check_protection() local
4145 entry = tmp_entry; in vm_map_check_protection()
4151 if (start < entry->start) in vm_map_check_protection()
4154 * Check protection associated with entry. in vm_map_check_protection()
4156 if ((entry->protection & protection) != protection) in vm_map_check_protection()
4158 /* go to next entry */ in vm_map_check_protection()
4159 start = entry->end; in vm_map_check_protection()
4160 entry = vm_map_entry_succ(entry); in vm_map_check_protection()
4169 * Copies a swap-backed object from an existing map entry to a
4220 * Copies the contents of the source entry to the destination
4221 * entry. The entries *must* be aligned properly.
4243 * If the source entry is marked needs_copy, it is already in vm_map_copy_entry()
4277 * src_entry, since the entry is in vm_map_copy_entry()
4279 * fake entry that is used to in vm_map_copy_entry()
4322 * Update the newly-forked vmspace each time a map entry is inherited
4328 vm_map_entry_t entry) in vmspace_map_entry_forked() argument
4333 if ((entry->eflags & MAP_ENTRY_GUARD) != 0) in vmspace_map_entry_forked()
4335 entrysize = entry->end - entry->start; in vmspace_map_entry_forked()
4337 if ((entry->eflags & MAP_ENTRY_GROWS_DOWN) != 0) { in vmspace_map_entry_forked()
4339 } else if (entry->start >= (vm_offset_t)vm1->vm_daddr && in vmspace_map_entry_forked()
4340 entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) { in vmspace_map_entry_forked()
4341 newend = MIN(entry->end, in vmspace_map_entry_forked()
4343 vm2->vm_dsize += btoc(newend - entry->start); in vmspace_map_entry_forked()
4344 } else if (entry->start >= (vm_offset_t)vm1->vm_taddr && in vmspace_map_entry_forked()
4345 entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) { in vmspace_map_entry_forked()
4346 newend = MIN(entry->end, in vmspace_map_entry_forked()
4348 vm2->vm_tsize += btoc(newend - entry->start); in vmspace_map_entry_forked()
4420 * Clone the entry, creating the shared object if in vmspace_fork()
4484 * Clone the entry, referencing the shared object. in vmspace_fork()
4499 * Insert the entry into the new map -- we know we're in vmspace_fork()
4516 * Clone the entry and link into the map. in vmspace_fork()
4521 * Copied entry is COW over the old object. in vmspace_fork()
4538 * Create a new anonymous mapping entry modelled from in vmspace_fork()
4662 ("Bad entry start/end for new stack entry")); in vm_map_stack_locked()
4664 ("new entry lacks MAP_ENTRY_GROWS_DOWN")); in vm_map_stack_locked()
4672 ("entry %p not gap %#x", gap_entry, gap_entry->eflags)); in vm_map_stack_locked()
4674 ("entry %p not stack gap %#x", gap_entry, in vm_map_stack_locked()
4680 * next_read of the gap entry to store in vm_map_stack_locked()
4695 * Attempts to grow a vm stack entry. Returns KERN_SUCCESS if we
5011 vm_map_entry_t entry; in vm_map_lookup() local
5032 entry = *out_entry; in vm_map_lookup()
5037 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { in vm_map_lookup()
5040 *var_map = map = entry->object.sub_map; in vm_map_lookup()
5048 prot = entry->protection; in vm_map_lookup()
5052 (entry->eflags & MAP_ENTRY_GUARD) != 0 && in vm_map_lookup()
5053 (entry->eflags & MAP_ENTRY_STACK_GAP) != 0 && in vm_map_lookup()
5054 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
5062 KASSERT((prot & VM_PROT_WRITE) == 0 || (entry->eflags & in vm_map_lookup()
5065 ("entry %p flags %x", entry, entry->eflags)); in vm_map_lookup()
5067 (entry->max_protection & VM_PROT_WRITE) == 0 && in vm_map_lookup()
5068 (entry->eflags & MAP_ENTRY_COW) == 0) { in vm_map_lookup()
5077 *wired = (entry->wired_count != 0); in vm_map_lookup()
5079 fault_type = entry->protection; in vm_map_lookup()
5080 size = entry->end - entry->start; in vm_map_lookup()
5083 * If the entry was copy-on-write, we either ... in vm_map_lookup()
5085 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { in vm_map_lookup()
5104 if (entry->cred == NULL) { in vm_map_lookup()
5116 entry->cred = cred; in vm_map_lookup()
5118 eobject = entry->object.vm_object; in vm_map_lookup()
5119 vm_object_shadow(&entry->object.vm_object, in vm_map_lookup()
5120 &entry->offset, size, entry->cred, false); in vm_map_lookup()
5121 if (eobject == entry->object.vm_object) { in vm_map_lookup()
5125 swap_release_by_cred(size, entry->cred); in vm_map_lookup()
5126 crfree(entry->cred); in vm_map_lookup()
5128 entry->cred = NULL; in vm_map_lookup()
5129 entry->eflags &= ~MAP_ENTRY_NEEDS_COPY; in vm_map_lookup()
5144 if (entry->object.vm_object == NULL && !vm_map_is_system(map)) { in vm_map_lookup()
5147 entry->object.vm_object = vm_object_allocate_anon(atop(size), in vm_map_lookup()
5148 NULL, entry->cred); in vm_map_lookup()
5149 entry->offset = 0; in vm_map_lookup()
5150 entry->cred = NULL; in vm_map_lookup()
5155 * Return the object/offset from this entry. If the entry was in vm_map_lookup()
5158 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); in vm_map_lookup()
5159 *object = entry->object.vm_object; in vm_map_lookup()
5181 vm_map_entry_t entry; in vm_map_lookup_locked() local
5192 entry = *out_entry; in vm_map_lookup_locked()
5195 * Fail if the entry refers to a submap. in vm_map_lookup_locked()
5197 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) in vm_map_lookup_locked()
5203 prot = entry->protection; in vm_map_lookup_locked()
5212 *wired = (entry->wired_count != 0); in vm_map_lookup_locked()
5214 fault_type = entry->protection; in vm_map_lookup_locked()
5216 if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { in vm_map_lookup_locked()
5218 * Fail if the entry was copy-on-write for a write fault. in vm_map_lookup_locked()
5232 if (entry->object.vm_object == NULL && !vm_map_is_system(map)) in vm_map_lookup_locked()
5236 * Return the object/offset from this entry. If the entry was in vm_map_lookup_locked()
5239 *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); in vm_map_lookup_locked()
5240 *object = entry->object.vm_object; in vm_map_lookup_locked()
5253 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
5293 vm_map_entry_t entry, prev; in _vm_map_assert_consistent() local
5304 VM_MAP_ENTRY_FOREACH(entry, map) { in _vm_map_assert_consistent()
5305 KASSERT(prev->end <= entry->start, in _vm_map_assert_consistent()
5307 (uintmax_t)prev->end, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5308 KASSERT(entry->start < entry->end, in _vm_map_assert_consistent()
5310 (uintmax_t)entry->start, (uintmax_t)entry->end)); in _vm_map_assert_consistent()
5311 KASSERT(entry->left == header || in _vm_map_assert_consistent()
5312 entry->left->start < entry->start, in _vm_map_assert_consistent()
5314 (uintmax_t)entry->left->start, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5315 KASSERT(entry->right == header || in _vm_map_assert_consistent()
5316 entry->start < entry->right->start, in _vm_map_assert_consistent()
5318 (uintmax_t)entry->start, (uintmax_t)entry->right->start)); in _vm_map_assert_consistent()
5322 if (entry->start < cur->start) { in _vm_map_assert_consistent()
5327 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5328 } else if (cur->end <= entry->start) { in _vm_map_assert_consistent()
5333 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5335 KASSERT(cur == entry, in _vm_map_assert_consistent()
5337 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5341 max_left = vm_map_entry_max_free_left(entry, lbound); in _vm_map_assert_consistent()
5342 max_right = vm_map_entry_max_free_right(entry, ubound); in _vm_map_assert_consistent()
5343 KASSERT(entry->max_free == vm_size_max(max_left, max_right), in _vm_map_assert_consistent()
5345 (uintmax_t)entry->max_free, in _vm_map_assert_consistent()
5347 prev = entry; in _vm_map_assert_consistent()
5349 KASSERT(prev->end <= entry->start, in _vm_map_assert_consistent()
5351 (uintmax_t)prev->end, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5364 vm_map_entry_t entry, prev; in vm_map_print() local
5372 VM_MAP_ENTRY_FOREACH(entry, map) { in vm_map_print()
5373 db_iprintf("map entry %p: start=%p, end=%p, eflags=%#x, \n", in vm_map_print()
5374 (void *)entry, (void *)entry->start, (void *)entry->end, in vm_map_print()
5375 entry->eflags); in vm_map_print()
5381 entry->protection, in vm_map_print()
5382 entry->max_protection, in vm_map_print()
5384 entry->inheritance]); in vm_map_print()
5385 if (entry->wired_count != 0) in vm_map_print()
5388 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { in vm_map_print()
5390 (void *)entry->object.sub_map, in vm_map_print()
5391 (uintmax_t)entry->offset); in vm_map_print()
5394 entry->object.sub_map) { in vm_map_print()
5396 vm_map_print((vm_map_t)entry->object.sub_map); in vm_map_print()
5400 if (entry->cred != NULL) in vm_map_print()
5401 db_printf(", ruid %d", entry->cred->cr_ruid); in vm_map_print()
5403 (void *)entry->object.vm_object, in vm_map_print()
5404 (uintmax_t)entry->offset); in vm_map_print()
5405 if (entry->object.vm_object && entry->object.vm_object->cred) in vm_map_print()
5407 entry->object.vm_object->cred->cr_ruid); in vm_map_print()
5408 if (entry->eflags & MAP_ENTRY_COW) in vm_map_print()
5410 (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done"); in vm_map_print()
5415 entry->object.vm_object) { in vm_map_print()
5418 entry->object.vm_object, in vm_map_print()
5423 prev = entry; in vm_map_print()