47 	struct extent_map *em;  in btrfs_alloc_extent_map()  local
48 	em = kmem_cache_zalloc(extent_map_cache, GFP_NOFS);  in btrfs_alloc_extent_map()
49 	if (!em)  in btrfs_alloc_extent_map()
51 	RB_CLEAR_NODE(&em->rb_node);  in btrfs_alloc_extent_map()
52 	refcount_set(&em->refs, 1);  in btrfs_alloc_extent_map()
53 	INIT_LIST_HEAD(&em->list);  in btrfs_alloc_extent_map()
54 	return em;  in btrfs_alloc_extent_map()
61 void btrfs_free_extent_map(struct extent_map *em)  in btrfs_free_extent_map()  argument
63 	if (!em)  in btrfs_free_extent_map()
65 	if (refcount_dec_and_test(&em->refs)) {  in btrfs_free_extent_map()
66 		WARN_ON(btrfs_extent_map_in_tree(em));  in btrfs_free_extent_map()
67 		WARN_ON(!list_empty(&em->list));  in btrfs_free_extent_map()
68 		kmem_cache_free(extent_map_cache, em);  in btrfs_free_extent_map()
80 static void remove_em(struct btrfs_inode *inode, struct extent_map *em)  in remove_em()  argument
84 	rb_erase(&em->rb_node, &inode->extent_tree.root);  in remove_em()
85 	RB_CLEAR_NODE(&em->rb_node);  in remove_em()
91 static int tree_insert(struct rb_root *root, struct extent_map *em)  in tree_insert()  argument
97 	u64 end = range_end(em->start, em->len);  in tree_insert()
103 		if (em->start < entry->start)  in tree_insert()
105 		else if (em->start >= btrfs_extent_map_end(entry))  in tree_insert()
112 	while (parent && em->start >= btrfs_extent_map_end(entry)) {  in tree_insert()
117 		if (end > entry->start && em->start < btrfs_extent_map_end(entry))  in tree_insert()
122 	while (parent && em->start < entry->start) {  in tree_insert()
127 		if (end > entry->start && em->start < btrfs_extent_map_end(entry))  in tree_insert()
130 	rb_link_node(&em->rb_node, orig_parent, p);  in tree_insert()
131 	rb_insert_color(&em->rb_node, root);  in tree_insert()
189 static inline u64 extent_map_block_len(const struct extent_map *em)  in extent_map_block_len()  argument
191 	if (btrfs_extent_map_is_compressed(em))  in extent_map_block_len()
192 		return em->disk_num_bytes;  in extent_map_block_len()
193 	return em->len;  in extent_map_block_len()
196 static inline u64 extent_map_block_end(const struct extent_map *em)  in extent_map_block_end()  argument
198 	const u64 block_start = btrfs_extent_map_block_start(em);  in extent_map_block_end()
199 	const u64 block_end = block_start + extent_map_block_len(em);  in extent_map_block_end()
207 static bool can_merge_extent_map(const struct extent_map *em)  in can_merge_extent_map()  argument
209 	if (em->flags & EXTENT_FLAG_PINNED)  in can_merge_extent_map()
213 	if (btrfs_extent_map_is_compressed(em))  in can_merge_extent_map()
216 	if (em->flags & EXTENT_FLAG_LOGGING)  in can_merge_extent_map()
224 	if (!list_empty(&em->list))  in can_merge_extent_map()
308 			    struct extent_map *em)  in dump_extent_map()  argument
314 		prefix, em->start, em->len, em->disk_bytenr, em->disk_num_bytes,  in dump_extent_map()
315 		em->ram_bytes, em->offset, em->flags);  in dump_extent_map()
320 static void validate_extent_map(struct btrfs_fs_info *fs_info, struct extent_map *em)  in validate_extent_map()  argument
327 	if (!IS_ALIGNED(em->start, blocksize) ||  in validate_extent_map()
328 	    !IS_ALIGNED(em->len, blocksize))  in validate_extent_map()
329 		dump_extent_map(fs_info, "unaligned start offset or length members", em);  in validate_extent_map()
331 	if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {  in validate_extent_map()
332 		if (em->disk_num_bytes == 0)  in validate_extent_map()
333 			dump_extent_map(fs_info, "zero disk_num_bytes", em);  in validate_extent_map()
334 		if (em->offset + em->len > em->ram_bytes)  in validate_extent_map()
335 			dump_extent_map(fs_info, "ram_bytes too small", em);  in validate_extent_map()
336 		if (em->offset + em->len > em->disk_num_bytes &&  in validate_extent_map()
337 		    !btrfs_extent_map_is_compressed(em))  in validate_extent_map()
338 			dump_extent_map(fs_info, "disk_num_bytes too small", em);  in validate_extent_map()
339 		if (!btrfs_extent_map_is_compressed(em) &&  in validate_extent_map()
340 		    em->ram_bytes != em->disk_num_bytes)  in validate_extent_map()
343 					em);  in validate_extent_map()
344 		if (!IS_ALIGNED(em->disk_bytenr, blocksize) ||  in validate_extent_map()
345 		    !IS_ALIGNED(em->disk_num_bytes, blocksize) ||  in validate_extent_map()
346 		    !IS_ALIGNED(em->offset, blocksize) ||  in validate_extent_map()
347 		    !IS_ALIGNED(em->ram_bytes, blocksize))  in validate_extent_map()
348 			dump_extent_map(fs_info, "unaligned members", em);  in validate_extent_map()
349 	} else if (em->offset) {  in validate_extent_map()
350 		dump_extent_map(fs_info, "non-zero offset for hole/inline", em);  in validate_extent_map()
354 static void try_merge_map(struct btrfs_inode *inode, struct extent_map *em)  in try_merge_map()  argument
368 	if (refcount_read(&em->refs) > 2)  in try_merge_map()
371 	if (!can_merge_extent_map(em))  in try_merge_map()
374 	if (em->start != 0) {  in try_merge_map()
375 		rb = rb_prev(&em->rb_node);  in try_merge_map()
378 		if (rb && can_merge_extent_map(merge) && mergeable_maps(merge, em)) {  in try_merge_map()
379 			em->start = merge->start;  in try_merge_map()
380 			em->len += merge->len;  in try_merge_map()
381 			em->generation = max(em->generation, merge->generation);  in try_merge_map()
383 			if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)  in try_merge_map()
384 				merge_ondisk_extents(merge, em, em);  in try_merge_map()
385 			em->flags |= EXTENT_FLAG_MERGED;  in try_merge_map()
387 			validate_extent_map(fs_info, em);  in try_merge_map()
393 	rb = rb_next(&em->rb_node);  in try_merge_map()
396 	if (rb && can_merge_extent_map(merge) && mergeable_maps(em, merge)) {  in try_merge_map()
397 		em->len += merge->len;  in try_merge_map()
398 		if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)  in try_merge_map()
399 			merge_ondisk_extents(em, merge, em);  in try_merge_map()
400 		validate_extent_map(fs_info, em);  in try_merge_map()
401 		em->generation = max(em->generation, merge->generation);  in try_merge_map()
402 		em->flags |= EXTENT_FLAG_MERGED;  in try_merge_map()
429 	struct extent_map *em;  in btrfs_unpin_extent_cache()  local
432 	em = btrfs_lookup_extent_mapping(tree, start, len);  in btrfs_unpin_extent_cache()
434 	if (WARN_ON(!em)) {  in btrfs_unpin_extent_cache()
443 	if (WARN_ON(em->start != start)) {  in btrfs_unpin_extent_cache()
447 			   em->start, start, start + len, gen);  in btrfs_unpin_extent_cache()
452 	em->generation = gen;  in btrfs_unpin_extent_cache()
453 	em->flags &= ~EXTENT_FLAG_PINNED;  in btrfs_unpin_extent_cache()
455 	try_merge_map(inode, em);  in btrfs_unpin_extent_cache()
459 	btrfs_free_extent_map(em);  in btrfs_unpin_extent_cache()
464 void btrfs_clear_em_logging(struct btrfs_inode *inode, struct extent_map *em)  in btrfs_clear_em_logging()  argument
468 	em->flags &= ~EXTENT_FLAG_LOGGING;  in btrfs_clear_em_logging()
469 	if (btrfs_extent_map_in_tree(em))  in btrfs_clear_em_logging()
470 		try_merge_map(inode, em);  in btrfs_clear_em_logging()
474 					struct extent_map *em,  in setup_extent_mapping()  argument
477 	refcount_inc(&em->refs);  in setup_extent_mapping()
479 	ASSERT(list_empty(&em->list));  in setup_extent_mapping()
482 		list_add(&em->list, &inode->extent_tree.modified_extents);  in setup_extent_mapping()
484 		try_merge_map(inode, em);  in setup_extent_mapping()
501 			      struct extent_map *em, bool modified)  in add_extent_mapping()  argument
510 	validate_extent_map(fs_info, em);  in add_extent_mapping()
511 	ret = tree_insert(&tree->root, em);  in add_extent_mapping()
515 	setup_extent_mapping(inode, em, modified);  in add_extent_mapping()
526 	struct extent_map *em;  in lookup_extent_mapping()  local
539 	em = rb_entry(rb_node, struct extent_map, rb_node);  in lookup_extent_mapping()
541 	if (strict && !(end > em->start && start < btrfs_extent_map_end(em)))  in lookup_extent_mapping()
544 	refcount_inc(&em->refs);  in lookup_extent_mapping()
545 	return em;  in lookup_extent_mapping()
593 void btrfs_remove_extent_mapping(struct btrfs_inode *inode, struct extent_map *em)  in btrfs_remove_extent_mapping()  argument
599 	WARN_ON(em->flags & EXTENT_FLAG_PINNED);  in btrfs_remove_extent_mapping()
600 	if (!(em->flags & EXTENT_FLAG_LOGGING))  in btrfs_remove_extent_mapping()
601 		list_del_init(&em->list);  in btrfs_remove_extent_mapping()
603 	remove_em(inode, em);  in btrfs_remove_extent_mapping()
628 static struct extent_map *next_extent_map(const struct extent_map *em)  in next_extent_map()  argument
632 	next = rb_next(&em->rb_node);  in next_extent_map()
638 static struct extent_map *prev_extent_map(struct extent_map *em)  in prev_extent_map()  argument
642 	prev = rb_prev(&em->rb_node);  in prev_extent_map()
656 					 struct extent_map *em,  in merge_extent_mapping()  argument
665 	if (map_start < em->start || map_start >= btrfs_extent_map_end(em))  in merge_extent_mapping()
676 	start = prev ? btrfs_extent_map_end(prev) : em->start;  in merge_extent_mapping()
677 	start = max_t(u64, start, em->start);  in merge_extent_mapping()
678 	end = next ? next->start : btrfs_extent_map_end(em);  in merge_extent_mapping()
679 	end = min_t(u64, end, btrfs_extent_map_end(em));  in merge_extent_mapping()
680 	start_diff = start - em->start;  in merge_extent_mapping()
681 	em->start = start;  in merge_extent_mapping()
682 	em->len = end - start;  in merge_extent_mapping()
683 	if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)  in merge_extent_mapping()
684 		em->offset += start_diff;  in merge_extent_mapping()
685 	return add_extent_mapping(inode, em, false);  in merge_extent_mapping()
712 	struct extent_map *em = *em_in;  in btrfs_add_extent_mapping()  local
719 	if (em->disk_bytenr == EXTENT_MAP_INLINE)  in btrfs_add_extent_mapping()
720 		ASSERT(em->start == 0);  in btrfs_add_extent_mapping()
722 	ret = add_extent_mapping(inode, em, false);  in btrfs_add_extent_mapping()
732 		trace_btrfs_handle_em_exist(fs_info, existing, em, start, len);  in btrfs_add_extent_mapping()
740 			btrfs_free_extent_map(em);  in btrfs_add_extent_mapping()
744 			u64 orig_start = em->start;  in btrfs_add_extent_mapping()
745 			u64 orig_len = em->len;  in btrfs_add_extent_mapping()
751 			ret = merge_extent_mapping(inode, existing, em, start);  in btrfs_add_extent_mapping()
753 				btrfs_free_extent_map(em);  in btrfs_add_extent_mapping()
781 		struct extent_map *em;  in drop_all_extent_maps_fast()  local
784 		em = rb_entry(node, struct extent_map, rb_node);  in drop_all_extent_maps_fast()
785 		em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING);  in drop_all_extent_maps_fast()
786 		btrfs_remove_extent_mapping(inode, em);  in drop_all_extent_maps_fast()
787 		btrfs_free_extent_map(em);  in drop_all_extent_maps_fast()
816 	struct extent_map *em;  in btrfs_drop_extent_map_range()  local
844 	em = btrfs_lookup_extent_mapping(em_tree, start, len);  in btrfs_drop_extent_map_range()
846 	while (em) {  in btrfs_drop_extent_map_range()
848 		const u64 em_end = btrfs_extent_map_end(em);  in btrfs_drop_extent_map_range()
855 			next_em = next_extent_map(em);  in btrfs_drop_extent_map_range()
864 		if (skip_pinned && (em->flags & EXTENT_FLAG_PINNED)) {  in btrfs_drop_extent_map_range()
869 		flags = em->flags;  in btrfs_drop_extent_map_range()
875 		em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING);  in btrfs_drop_extent_map_range()
876 		modified = !list_empty(&em->list);  in btrfs_drop_extent_map_range()
882 		if (em->start >= start && em_end <= end)  in btrfs_drop_extent_map_range()
885 		gen = em->generation;  in btrfs_drop_extent_map_range()
887 		if (em->start < start) {  in btrfs_drop_extent_map_range()
894 			split->start = em->start;  in btrfs_drop_extent_map_range()
895 			split->len = start - em->start;  in btrfs_drop_extent_map_range()
897 			if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {  in btrfs_drop_extent_map_range()
898 				split->disk_bytenr = em->disk_bytenr;  in btrfs_drop_extent_map_range()
899 				split->disk_num_bytes = em->disk_num_bytes;  in btrfs_drop_extent_map_range()
900 				split->offset = em->offset;  in btrfs_drop_extent_map_range()
901 				split->ram_bytes = em->ram_bytes;  in btrfs_drop_extent_map_range()
903 				split->disk_bytenr = em->disk_bytenr;  in btrfs_drop_extent_map_range()
911 			replace_extent_mapping(inode, em, split, modified);  in btrfs_drop_extent_map_range()
925 			split->disk_bytenr = em->disk_bytenr;  in btrfs_drop_extent_map_range()
929 			if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {  in btrfs_drop_extent_map_range()
930 				split->disk_num_bytes = em->disk_num_bytes;  in btrfs_drop_extent_map_range()
931 				split->offset = em->offset + end - em->start;  in btrfs_drop_extent_map_range()
932 				split->ram_bytes = em->ram_bytes;  in btrfs_drop_extent_map_range()
939 			if (btrfs_extent_map_in_tree(em)) {  in btrfs_drop_extent_map_range()
940 				replace_extent_mapping(inode, em, split, modified);  in btrfs_drop_extent_map_range()
954 		if (btrfs_extent_map_in_tree(em)) {  in btrfs_drop_extent_map_range()
975 			if ((em->start < start || em_end > end) && modified) {  in btrfs_drop_extent_map_range()
979 			btrfs_remove_extent_mapping(inode, em);  in btrfs_drop_extent_map_range()
986 		btrfs_free_extent_map(em);  in btrfs_drop_extent_map_range()
989 		btrfs_free_extent_map(em);  in btrfs_drop_extent_map_range()
991 		em = next_em;  in btrfs_drop_extent_map_range()
1051 	struct extent_map *em;  in btrfs_split_extent_map()  local
1071 	em = btrfs_lookup_extent_mapping(em_tree, start, len);  in btrfs_split_extent_map()
1072 	if (unlikely(!em)) {  in btrfs_split_extent_map()
1077 	ASSERT(em->len == len);  in btrfs_split_extent_map()
1078 	ASSERT(!btrfs_extent_map_is_compressed(em));  in btrfs_split_extent_map()
1079 	ASSERT(em->disk_bytenr < EXTENT_MAP_LAST_BYTE);  in btrfs_split_extent_map()
1080 	ASSERT(em->flags & EXTENT_FLAG_PINNED);  in btrfs_split_extent_map()
1081 	ASSERT(!(em->flags & EXTENT_FLAG_LOGGING));  in btrfs_split_extent_map()
1082 	ASSERT(!list_empty(&em->list));  in btrfs_split_extent_map()
1084 	flags = em->flags;  in btrfs_split_extent_map()
1085 	em->flags &= ~EXTENT_FLAG_PINNED;  in btrfs_split_extent_map()
1088 	split_pre->start = em->start;  in btrfs_split_extent_map()
1095 	split_pre->generation = em->generation;  in btrfs_split_extent_map()
1097 	replace_extent_mapping(inode, em, split_pre, true);  in btrfs_split_extent_map()
1105 	split_mid->start = em->start + pre;  in btrfs_split_extent_map()
1106 	split_mid->len = em->len - pre;  in btrfs_split_extent_map()
1107 	split_mid->disk_bytenr = btrfs_extent_map_block_start(em) + pre;  in btrfs_split_extent_map()
1112 	split_mid->generation = em->generation;  in btrfs_split_extent_map()
1116 	btrfs_free_extent_map(em);  in btrfs_split_extent_map()
1118 	btrfs_free_extent_map(em);  in btrfs_split_extent_map()
1164 		struct extent_map *em;  in btrfs_scan_inode()  local
1166 		em = rb_entry(node, struct extent_map, rb_node);  in btrfs_scan_inode()
1169 		if (em->flags & EXTENT_FLAG_PINNED)  in btrfs_scan_inode()
1179 		if (!list_empty(&em->list) && em->generation >= cur_fs_gen)  in btrfs_scan_inode()
1182 		btrfs_remove_extent_mapping(inode, em);  in btrfs_scan_inode()
1183 		trace_btrfs_extent_map_shrinker_remove_em(inode, em);  in btrfs_scan_inode()
1185 		btrfs_free_extent_map(em);  in btrfs_scan_inode()