Lines Matching full:ctl
36 static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
38 static int link_free_space(struct btrfs_free_space_ctl *ctl,
40 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
637 static void merge_space_tree(struct btrfs_free_space_ctl *ctl) in merge_space_tree() argument
643 spin_lock(&ctl->tree_lock); in merge_space_tree()
644 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in merge_space_tree()
651 unlink_free_space(ctl, prev); in merge_space_tree()
652 unlink_free_space(ctl, e); in merge_space_tree()
655 link_free_space(ctl, prev); in merge_space_tree()
657 spin_unlock(&ctl->tree_lock); in merge_space_tree()
663 spin_unlock(&ctl->tree_lock); in merge_space_tree()
667 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
772 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
773 ret = link_free_space(ctl, e); in __load_free_space_cache()
774 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
791 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
792 ret = link_free_space(ctl, e); in __load_free_space_cache()
793 ctl->total_bitmaps++; in __load_free_space_cache()
794 ctl->op->recalc_thresholds(ctl); in __load_free_space_cache()
795 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
819 e->bitmap_extents = count_bitmap_extents(ctl, e); in __load_free_space_cache()
821 ctl->discardable_extents[BTRFS_STAT_CURR] += in __load_free_space_cache()
823 ctl->discardable_bytes[BTRFS_STAT_CURR] += e->bytes; in __load_free_space_cache()
828 merge_space_tree(ctl); in __load_free_space_cache()
831 btrfs_discard_update_discardable(ctl->private, ctl); in __load_free_space_cache()
836 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
843 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
901 ret = __load_free_space_cache(fs_info->tree_root, inode, ctl, in load_free_space_cache()
907 spin_lock(&ctl->tree_lock); in load_free_space_cache()
908 matched = (ctl->free_space == (block_group->length - used - in load_free_space_cache()
910 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
913 __btrfs_remove_free_space_cache(ctl); in load_free_space_cache()
938 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
946 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
998 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1243 * @ctl - the free space cache we are going to write out
1252 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1297 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1299 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1300 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1324 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1325 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1367 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1368 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1395 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1410 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, in btrfs_write_out_cache()
1445 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1451 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1452 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1455 bitmap_start += ctl->start; in offset_to_bitmap()
1519 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1522 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1608 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1619 ctl->unit > offset) in tree_search_offset()
1635 __unlink_free_space(struct btrfs_free_space_ctl *ctl, in __unlink_free_space() argument
1638 rb_erase(&info->offset_index, &ctl->free_space_offset); in __unlink_free_space()
1639 ctl->free_extents--; in __unlink_free_space()
1642 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __unlink_free_space()
1643 ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes; in __unlink_free_space()
1647 static void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1650 __unlink_free_space(ctl, info); in unlink_free_space()
1651 ctl->free_space -= info->bytes; in unlink_free_space()
1654 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1660 ret = tree_insert_offset(&ctl->free_space_offset, info->offset, in link_free_space()
1666 ctl->discardable_extents[BTRFS_STAT_CURR]++; in link_free_space()
1667 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in link_free_space()
1670 ctl->free_space += info->bytes; in link_free_space()
1671 ctl->free_extents++; in link_free_space()
1675 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
1677 struct btrfs_block_group *block_group = ctl->private; in recalculate_thresholds()
1682 u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
1687 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
1700 bitmap_bytes = ctl->total_bitmaps * ctl->unit; in recalculate_thresholds()
1709 ctl->extents_thresh = in recalculate_thresholds()
1713 static inline void __bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in __bitmap_clear_bits() argument
1720 start = offset_to_bit(info->offset, ctl->unit, offset); in __bitmap_clear_bits()
1721 count = bytes_to_bits(bytes, ctl->unit); in __bitmap_clear_bits()
1728 if (info->max_extent_size > ctl->unit) in __bitmap_clear_bits()
1739 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in __bitmap_clear_bits()
1740 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in __bitmap_clear_bits()
1744 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1748 __bitmap_clear_bits(ctl, info, offset, bytes); in bitmap_clear_bits()
1749 ctl->free_space -= bytes; in bitmap_clear_bits()
1752 static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in bitmap_set_bits() argument
1759 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_set_bits()
1760 count = bytes_to_bits(bytes, ctl->unit); in bitmap_set_bits()
1767 ctl->free_space += bytes; in bitmap_set_bits()
1777 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in bitmap_set_bits()
1778 ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes; in bitmap_set_bits()
1786 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1807 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1809 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1829 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1830 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
1834 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
1848 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
1857 if (!ctl->free_space_offset.rb_node) in find_free_space()
1860 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), 0, 1); in find_free_space()
1876 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
1878 tmp = tmp * align + ctl->start; in find_free_space()
1894 ret = search_bitmap(ctl, entry, &tmp, &size, true); in find_free_space()
1915 static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl, in count_bitmap_extents() argument
1918 struct btrfs_block_group *block_group = ctl->private; in count_bitmap_extents()
1928 bytes -= (rs - re) * ctl->unit; in count_bitmap_extents()
1938 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
1941 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
1945 link_free_space(ctl, info); in add_new_bitmap()
1946 ctl->total_bitmaps++; in add_new_bitmap()
1948 ctl->op->recalc_thresholds(ctl); in add_new_bitmap()
1951 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
1961 ctl->discardable_extents[BTRFS_STAT_CURR] -= in free_bitmap()
1963 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes; in free_bitmap()
1966 unlink_free_space(ctl, bitmap_info); in free_bitmap()
1969 ctl->total_bitmaps--; in free_bitmap()
1970 ctl->op->recalc_thresholds(ctl); in free_bitmap()
1973 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
1982 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
1991 search_bytes = ctl->unit; in remove_from_bitmap()
1993 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, in remove_from_bitmap()
2004 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes); in remove_from_bitmap()
2011 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2037 search_bytes = ctl->unit; in remove_from_bitmap()
2038 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
2045 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2050 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
2063 ctl->discardable_extents[BTRFS_STAT_CURR] += in add_bytes_to_bitmap()
2065 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in add_bytes_to_bitmap()
2070 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
2074 bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
2086 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
2089 struct btrfs_block_group *block_group = ctl->private; in use_bitmap()
2106 if (!forced && ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
2115 if (ctl->free_extents * 3 <= ctl->extents_thresh) in use_bitmap()
2130 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) in use_bitmap()
2141 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2155 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2158 if (ctl->op == &free_space_op) in insert_into_bitmap()
2159 block_group = ctl->private; in insert_into_bitmap()
2187 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2188 bytes_added = add_bytes_to_bitmap(ctl, entry, offset, in insert_into_bitmap()
2201 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2208 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in insert_into_bitmap()
2222 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2227 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2234 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2244 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2279 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2294 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2299 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2305 unlink_free_space(ctl, right_info); in try_merge_free_space()
2307 __unlink_free_space(ctl, right_info); in try_merge_free_space()
2318 unlink_free_space(ctl, left_info); in try_merge_free_space()
2320 __unlink_free_space(ctl, left_info); in try_merge_free_space()
2330 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2338 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2341 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2345 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2349 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2357 bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2359 __bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2362 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2367 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2378 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2383 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2386 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2390 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2402 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2404 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2414 bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2416 __bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2419 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2435 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2446 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2450 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2451 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2452 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2456 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2461 struct btrfs_free_space_ctl *ctl, in __btrfs_add_free_space() argument
2465 struct btrfs_block_group *block_group = ctl->private; in __btrfs_add_free_space()
2479 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2481 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2489 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2503 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2507 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2511 btrfs_discard_update_discardable(block_group, ctl); in __btrfs_add_free_space()
2512 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2562 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2567 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2574 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2580 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2595 unlink_free_space(ctl, info); in btrfs_remove_free_space()
2602 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2615 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2629 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2631 ret = __btrfs_add_free_space(block_group->fs_info, ctl, in btrfs_remove_free_space()
2640 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2646 btrfs_discard_update_discardable(block_group, ctl); in btrfs_remove_free_space()
2647 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2656 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2661 spin_lock(&ctl->tree_lock); in btrfs_dump_free_space()
2662 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2670 spin_unlock(&ctl->tree_lock); in btrfs_dump_free_space()
2680 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_init_free_space_ctl() local
2682 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2683 ctl->unit = fs_info->sectorsize; in btrfs_init_free_space_ctl()
2684 ctl->start = block_group->start; in btrfs_init_free_space_ctl()
2685 ctl->private = block_group; in btrfs_init_free_space_ctl()
2686 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2687 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2688 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2695 ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); in btrfs_init_free_space_ctl()
2708 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2733 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __btrfs_return_cluster_to_free_space()
2734 ctl->discardable_bytes[BTRFS_STAT_CURR] -= in __btrfs_return_cluster_to_free_space()
2738 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2739 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2743 ctl->discardable_extents[BTRFS_STAT_CURR]++; in __btrfs_return_cluster_to_free_space()
2744 ctl->discardable_bytes[BTRFS_STAT_CURR] += in __btrfs_return_cluster_to_free_space()
2748 tree_insert_offset(&ctl->free_space_offset, in __btrfs_return_cluster_to_free_space()
2759 struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache_locked() argument
2764 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache_locked()
2767 unlink_free_space(ctl, info); in __btrfs_remove_free_space_cache_locked()
2770 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache_locked()
2773 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache_locked()
2777 void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
2779 spin_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2780 __btrfs_remove_free_space_cache_locked(ctl); in __btrfs_remove_free_space_cache()
2781 if (ctl->private) in __btrfs_remove_free_space_cache()
2782 btrfs_discard_update_discardable(ctl->private, ctl); in __btrfs_remove_free_space_cache()
2783 spin_unlock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2788 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
2792 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2801 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2803 __btrfs_remove_free_space_cache_locked(ctl); in btrfs_remove_free_space_cache()
2804 btrfs_discard_update_discardable(block_group, ctl); in btrfs_remove_free_space_cache()
2805 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2817 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_is_free_space_trimmed() local
2822 spin_lock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
2823 node = rb_first(&ctl->free_space_offset); in btrfs_is_free_space_trimmed()
2836 spin_unlock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
2844 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
2854 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2855 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
2862 bitmap_clear_bits(ctl, entry, offset, bytes); in btrfs_find_space_for_alloc()
2868 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
2870 unlink_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2885 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2888 btrfs_discard_update_discardable(block_group, ctl); in btrfs_find_space_for_alloc()
2889 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2892 __btrfs_add_free_space(block_group->fs_info, ctl, in btrfs_find_space_for_alloc()
2910 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
2928 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
2931 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2933 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2947 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
2956 err = search_bitmap(ctl, entry, &search_start, &search_bytes, true); in btrfs_alloc_from_bitmap()
2964 __bitmap_clear_bits(ctl, entry, ret, bytes); in btrfs_alloc_from_bitmap()
2978 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
3043 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3048 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
3050 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in btrfs_alloc_from_cluster()
3052 ctl->free_extents--; in btrfs_alloc_from_cluster()
3056 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
3057 ctl->op->recalc_thresholds(ctl); in btrfs_alloc_from_cluster()
3059 ctl->discardable_extents[BTRFS_STAT_CURR]--; in btrfs_alloc_from_cluster()
3064 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3075 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
3086 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
3088 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
3089 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
3115 entry->max_extent_size = (u64)max_bits * ctl->unit; in btrfs_bitmap_cluster()
3126 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
3127 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
3134 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
3135 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
3141 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
3156 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
3165 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
3225 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
3247 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
3250 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
3252 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
3263 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
3297 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3320 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3326 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3327 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3364 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3391 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3415 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3417 __btrfs_add_free_space(fs_info, ctl, reserved_start, in do_trimming()
3421 __btrfs_add_free_space(fs_info, ctl, end, reserved_end - end, in do_trimming()
3423 __btrfs_add_free_space(fs_info, ctl, start, bytes, trim_state); in do_trimming()
3425 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3450 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3463 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3464 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3466 if (ctl->free_space < minlen) in trim_no_bitmap()
3469 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3493 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3494 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3497 unlink_free_space(ctl, entry); in trim_no_bitmap()
3510 link_free_space(ctl, entry); in trim_no_bitmap()
3518 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3519 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3523 unlink_free_space(ctl, entry); in trim_no_bitmap()
3527 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3530 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3531 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3558 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3559 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3578 static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset) in reset_trimming_bitmap() argument
3582 spin_lock(&ctl->tree_lock); in reset_trimming_bitmap()
3583 entry = tree_search_offset(ctl, offset, 1, 0); in reset_trimming_bitmap()
3586 ctl->discardable_extents[BTRFS_STAT_CURR] += in reset_trimming_bitmap()
3588 ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes; in reset_trimming_bitmap()
3593 spin_unlock(&ctl->tree_lock); in reset_trimming_bitmap()
3596 static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl, in end_trimming_bitmap() argument
3601 ctl->discardable_extents[BTRFS_STAT_CURR] -= in end_trimming_bitmap()
3603 ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes; in end_trimming_bitmap()
3616 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3621 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3628 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3629 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3631 if (ctl->free_space < minlen) { in trim_bitmaps()
3634 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3635 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3639 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3650 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3651 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3666 ret2 = search_bitmap(ctl, entry, &start, &bytes, false); in trim_bitmaps()
3673 end_trimming_bitmap(ctl, entry); in trim_bitmaps()
3676 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3677 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3687 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3688 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3694 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3695 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3710 bitmap_clear_bits(ctl, entry, start, bytes); in trim_bitmaps()
3712 free_bitmap(ctl, entry); in trim_bitmaps()
3714 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3717 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3718 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3723 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3730 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3739 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3757 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_trim_block_group() local
3776 div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); in btrfs_trim_block_group()
3779 reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); in btrfs_trim_block_group()
3840 struct btrfs_free_space_ctl *ctl = fs_root->free_ino_ctl; in btrfs_find_ino_for_alloc() local
3844 spin_lock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3846 if (RB_EMPTY_ROOT(&ctl->free_space_offset)) in btrfs_find_ino_for_alloc()
3849 entry = rb_entry(rb_first(&ctl->free_space_offset), in btrfs_find_ino_for_alloc()
3855 unlink_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3861 link_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3867 ret = search_bitmap(ctl, entry, &offset, &count, true); in btrfs_find_ino_for_alloc()
3872 bitmap_clear_bits(ctl, entry, offset, 1); in btrfs_find_ino_for_alloc()
3874 free_bitmap(ctl, entry); in btrfs_find_ino_for_alloc()
3877 spin_unlock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3916 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in load_free_ino_cache() local
3943 ret = __load_free_space_cache(root, inode, ctl, path, 0); in load_free_ino_cache()
3962 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in btrfs_write_out_ino_cache() local
3971 ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans); in btrfs_write_out_ino_cache()
4005 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
4020 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4024 ret = link_free_space(ctl, info); in test_add_free_space_entry()
4025 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4039 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4040 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
4045 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
4050 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in test_add_free_space_entry()
4055 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4075 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
4079 spin_lock(&ctl->tree_lock); in test_check_exists()
4080 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
4082 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
4095 bit_bytes = ctl->unit; in test_check_exists()
4096 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); in test_check_exists()
4148 spin_unlock(&ctl->tree_lock); in test_check_exists()