43 #include "ordered-data.h"
84 	struct btrfs_ordered_extent *ordered;  member
411  * Cleanup all submitted ordered extents in specified range to handle errors
417  * to be released, which we want to happen only when finishing the ordered
439 		 * clear page Ordered and run the ordered extent accounting.  in btrfs_cleanup_ordered_extents()
441 		 * Here we can't just clear the Ordered bit, or  in btrfs_cleanup_ordered_extents()
443 		 * for the page range, and the ordered extent will never finish.  in btrfs_cleanup_ordered_extents()
455 		 * Here we just clear all Ordered bits for every page in the  in btrfs_cleanup_ordered_extents()
457 		 * the ordered extent accounting for the range.  in btrfs_cleanup_ordered_extents()
829  * This is done inside an ordered work queue, and the compression is spread
830  * across many cpus.  The actual IO submission is step two, and the ordered work
1112 	struct btrfs_ordered_extent *ordered;  in submit_one_async_extent()  local
1172 	ordered = btrfs_alloc_ordered_extent(inode, start,	/* file_offset */  in submit_one_async_extent()
1180 	if (IS_ERR(ordered)) {  in submit_one_async_extent()
1182 		ret = PTR_ERR(ordered);  in submit_one_async_extent()
1191 	btrfs_submit_compressed_write(ordered,  in submit_one_async_extent()
1258  * allocate extents on disk for the range, and create ordered data structs
1278  * while-loop, the ordered extents created in previous iterations are kept
1384 		struct btrfs_ordered_extent *ordered;  in cow_file_range()  local
1435 		ordered = btrfs_alloc_ordered_extent(inode, start, ram_size,  in cow_file_range()
1439 		if (IS_ERR(ordered)) {  in cow_file_range()
1440 			ret = PTR_ERR(ordered);  in cow_file_range()
1445 			ret = btrfs_reloc_clone_csums(ordered);  in cow_file_range()
1451 			 * at out_unlock label to free meta of this ordered  in cow_file_range()
1456 			 * skip current ordered extent.  in cow_file_range()
1463 		btrfs_put_ordered_extent(ordered);  in cow_file_range()
1472 		 * Do set the Ordered (Private2) bit so we know this page was  in cow_file_range()
1493 		 * free metadata of current ordered extent, we're OK to exit.  in cow_file_range()
1523 	 * For the range (1). We have already instantiated the ordered extents  in cow_file_range()
1543 	 * (or a subrange) and failed to create the respective ordered extent,  in cow_file_range()
1575  * Phase two of compressed writeback.  This is the ordered portion of the code,
2007 		struct btrfs_ordered_extent *ordered;  in run_delalloc_nocow()  local
2168 		ordered = btrfs_alloc_ordered_extent(inode, cur_offset,  in run_delalloc_nocow()
2176 		if (IS_ERR(ordered)) {  in run_delalloc_nocow()
2181 			ret = PTR_ERR(ordered);  in run_delalloc_nocow()
2189 			 * from freeing metadata of created ordered extent.  in run_delalloc_nocow()
2191 			ret = btrfs_reloc_clone_csums(ordered);  in run_delalloc_nocow()
2192 		btrfs_put_ordered_extent(ordered);  in run_delalloc_nocow()
2204 		 * handler, as metadata for created ordered extent will only  in run_delalloc_nocow()
2560 					struct btrfs_ordered_extent *ordered)  in btrfs_extract_ordered_extent()  argument
2567 	/* Must always be called for the beginning of an ordered extent. */  in btrfs_extract_ordered_extent()
2568 	if (WARN_ON_ONCE(start != ordered->disk_bytenr))  in btrfs_extract_ordered_extent()
2571 	/* No need to split if the ordered extent covers the entire bio. */  in btrfs_extract_ordered_extent()
2572 	if (ordered->disk_num_bytes == len) {  in btrfs_extract_ordered_extent()
2573 		refcount_inc(&ordered->refs);  in btrfs_extract_ordered_extent()
2574 		bbio->ordered = ordered;  in btrfs_extract_ordered_extent()
2582 	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) {  in btrfs_extract_ordered_extent()
2584 				       ordered->num_bytes, len,  in btrfs_extract_ordered_extent()
2585 				       ordered->disk_bytenr);  in btrfs_extract_ordered_extent()
2590 	new = btrfs_split_ordered_extent(ordered, len);  in btrfs_extract_ordered_extent()
2593 	bbio->ordered = new;  in btrfs_extract_ordered_extent()
2598  * given a list of ordered sums record them in the inode.  This happens
2698 	struct btrfs_ordered_extent *ordered;  in btrfs_writepage_fixup_worker()  local
2731 		 *    while waiting for the ordered extent.  in btrfs_writepage_fixup_worker()
2760 	/* already ordered? We're done */  in btrfs_writepage_fixup_worker()
2764 	ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);  in btrfs_writepage_fixup_worker()
2765 	if (ordered) {  in btrfs_writepage_fixup_worker()
2769 		btrfs_start_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2770 		btrfs_put_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2822  * properly happens and the data=ordered rules are followed.
2824  * In our case any range that doesn't have the ORDERED bit set
2826  * to fix it up.  The async helper will wait for ordered extents, set
2835 	/* This page has ordered extent covering it already */  in btrfs_writepage_cow_fixup()
2935 	 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.  in insert_reserved_file_extent()
3005 	 * For delalloc, when completing an ordered extent we update the inode's  in insert_ordered_extent_file_extent()
3008 	 * except if the ordered extent was truncated.  in insert_ordered_extent_file_extent()
3020  * As ordered data IO finishes, this gets called so we can finish
3021  * an ordered extent if the range of bytes in the file it covers are
3172 		 * If we failed to finish this ordered extent for any reason we  in btrfs_finish_one_ordered()
3173 		 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered  in btrfs_finish_one_ordered()
3177 		 * marking this ordered extent as failed.  in btrfs_finish_one_ordered()
3206 		 * If the ordered extent had an IOERR or something else went  in btrfs_finish_one_ordered()
3207 		 * wrong we need to return the space for this ordered extent  in btrfs_finish_one_ordered()
3233 			 * the ordered extent was created.  in btrfs_finish_one_ordered()
3243 	 * updating everything for this ordered extent.  in btrfs_finish_one_ordered()
3255 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)  in btrfs_finish_ordered_io()  argument
3257 	if (btrfs_is_zoned(btrfs_sb(ordered->inode->i_sb)) &&  in btrfs_finish_ordered_io()
3258 	    !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) &&  in btrfs_finish_ordered_io()
3259 	    list_empty(&ordered->bioc_list))  in btrfs_finish_ordered_io()
3260 		btrfs_finish_ordered_zoned(ordered);  in btrfs_finish_ordered_io()
3261 	return btrfs_finish_one_ordered(ordered);  in btrfs_finish_ordered_io()
4691 	struct btrfs_ordered_extent *ordered;  in btrfs_truncate_block()  local
4767 	ordered = btrfs_lookup_ordered_extent(inode, block_start);  in btrfs_truncate_block()
4768 	if (ordered) {  in btrfs_truncate_block()
4772 		btrfs_start_ordered_extent(ordered);  in btrfs_truncate_block()
4773 		btrfs_put_ordered_extent(ordered);  in btrfs_truncate_block()
5107  * the ordered io finishing, while we release here the extent_map and
6978 	struct btrfs_ordered_extent *ordered;  in btrfs_create_dio_extent()  local
6988 	ordered = btrfs_alloc_ordered_extent(inode, start, len, len,  in btrfs_create_dio_extent()
6993 	if (IS_ERR(ordered)) {  in btrfs_create_dio_extent()
6999 		em = ERR_CAST(ordered);  in btrfs_create_dio_extent()
7001 		ASSERT(!dio_data->ordered);  in btrfs_create_dio_extent()
7002 		dio_data->ordered = ordered;  in btrfs_create_dio_extent()
7075  *	 any ordered extents.
7183 	struct btrfs_ordered_extent *ordered;  in lock_extent_direct()  local
7196 		 * doing DIO to, so we need to make sure there's no ordered  in lock_extent_direct()
7199 		ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,  in lock_extent_direct()
7209 		if (!ordered &&  in lock_extent_direct()
7216 		if (ordered) {  in lock_extent_direct()
7218 				btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7223 			 * If we are doing a DIO read and the ordered extent we  in lock_extent_direct()
7229 			 * created an ordered extent for a previous extent map  in lock_extent_direct()
7238 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))  in lock_extent_direct()
7239 				btrfs_start_ordered_extent(ordered);  in lock_extent_direct()
7242 			btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7251 			 * ordered dio extent we created before but did not have  in lock_extent_direct()
7254 			 * ordered extent to complete while holding a lock on  in lock_extent_direct()
7435 	 * We have created our ordered extent, so we can now release our reservation  in btrfs_get_blocks_direct_write()
7722 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_iomap_end()
7730 		btrfs_put_ordered_extent(dio_data->ordered);  in btrfs_dio_iomap_end()
7731 		dio_data->ordered = NULL;  in btrfs_dio_iomap_end()
7754 		btrfs_finish_ordered_extent(bbio->ordered, NULL,  in btrfs_dio_end_io()
7786 	 * the ordered extent to match the submitted bio.  Hang on to the  in btrfs_dio_submit_io()
7789 	 * remaining pages is blocked on the outstanding ordered extent.  in btrfs_dio_submit_io()
7794 		ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);  in btrfs_dio_submit_io()
7796 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_submit_io()
7852 	 * complete and writeback to start. We also need to wait for ordered  in btrfs_fiemap()
7868 	 * triggering writeback and waiting for the completion of IO and ordered  in btrfs_fiemap()
7983 	 * We have folio locked so no new ordered extent can be created on this  in btrfs_invalidate_folio()
7987 	 * Furthermore, endio function won't skip folio which has Ordered  in btrfs_invalidate_folio()
7989 	 * invalidate_folio to do the same ordered extent accounting twice  in btrfs_invalidate_folio()
7993 	 * do double ordered extent accounting on the same folio.  in btrfs_invalidate_folio()
8008 	 * still safe to wait for ordered extent to finish.  in btrfs_invalidate_folio()
8020 		struct btrfs_ordered_extent *ordered;  in btrfs_invalidate_folio()  local
8025 		ordered = btrfs_lookup_first_ordered_range(inode, cur,  in btrfs_invalidate_folio()
8027 		if (!ordered) {  in btrfs_invalidate_folio()
8030 			 * No ordered extent covering this range, we are safe  in btrfs_invalidate_folio()
8036 		if (ordered->file_offset > cur) {  in btrfs_invalidate_folio()
8039 			 * covered by any ordered extent.  in btrfs_invalidate_folio()
8041 			 * the ordered extent in the next iteration.  in btrfs_invalidate_folio()
8043 			range_end = ordered->file_offset - 1;  in btrfs_invalidate_folio()
8048 		range_end = min(ordered->file_offset + ordered->num_bytes - 1,  in btrfs_invalidate_folio()
8054 			 * If Ordered (Private2) is cleared, it means endio has  in btrfs_invalidate_folio()
8065 		 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW  in btrfs_invalidate_folio()
8066 		 * here, must leave that up for the ordered extent completion.  in btrfs_invalidate_folio()
8078 		set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);  in btrfs_invalidate_folio()
8079 		ordered->truncated_len = min(ordered->truncated_len,  in btrfs_invalidate_folio()
8080 					     cur - ordered->file_offset);  in btrfs_invalidate_folio()
8084 		 * If the ordered extent has finished, we're safe to delete all  in btrfs_invalidate_folio()
8089 		if (btrfs_dec_test_ordered_pending(inode, &ordered,  in btrfs_invalidate_folio()
8091 			btrfs_finish_ordered_io(ordered);  in btrfs_invalidate_folio()
8093 			 * The ordered extent has finished, now we're again  in btrfs_invalidate_folio()
8099 		if (ordered)  in btrfs_invalidate_folio()
8100 			btrfs_put_ordered_extent(ordered);  in btrfs_invalidate_folio()
8126 	 * We have iterated through all ordered extents of the page, the page  in btrfs_invalidate_folio()
8127 	 * should not have Ordered (Private2) anymore, or the above iteration  in btrfs_invalidate_folio()
8159 	struct btrfs_ordered_extent *ordered;  in btrfs_page_mkwrite()  local
8224 	 * we can't set the delalloc bits if there are pending ordered  in btrfs_page_mkwrite()
8227 	ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,  in btrfs_page_mkwrite()
8229 	if (ordered) {  in btrfs_page_mkwrite()
8233 		btrfs_start_ordered_extent(ordered);  in btrfs_page_mkwrite()
8234 		btrfs_put_ordered_extent(ordered);  in btrfs_page_mkwrite()
8612 	struct btrfs_ordered_extent *ordered;  in btrfs_destroy_inode()  local
8638 	 * If this is a free space inode do not take the ordered extents lockdep  in btrfs_destroy_inode()
8644 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);  in btrfs_destroy_inode()
8645 		if (!ordered)  in btrfs_destroy_inode()
8649 				  "found ordered extent %llu %llu on inode cleanup",  in btrfs_destroy_inode()
8650 				  ordered->file_offset, ordered->num_bytes);  in btrfs_destroy_inode()
8655 			btrfs_remove_ordered_extent(inode, ordered);  in btrfs_destroy_inode()
8656 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
8657 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
10151 		struct btrfs_ordered_extent *ordered;  in btrfs_encoded_read()  local
10158 		ordered = btrfs_lookup_ordered_range(inode, start,  in btrfs_encoded_read()
10160 		if (!ordered)  in btrfs_encoded_read()
10162 		btrfs_put_ordered_extent(ordered);  in btrfs_encoded_read()
10274 	struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10394 		struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10405 		ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);  in btrfs_do_encoded_write()
10406 		if (!ordered &&  in btrfs_do_encoded_write()
10409 		if (ordered)  in btrfs_do_encoded_write()
10410 			btrfs_put_ordered_extent(ordered);  in btrfs_do_encoded_write()
10458 	ordered = btrfs_alloc_ordered_extent(inode, start, num_bytes, ram_bytes,  in btrfs_do_encoded_write()
10464 	if (IS_ERR(ordered)) {  in btrfs_do_encoded_write()
10466 		ret = PTR_ERR(ordered);  in btrfs_do_encoded_write()
10478 	btrfs_submit_compressed_write(ordered, pages, nr_pages, 0, false);  in btrfs_do_encoded_write()
10933  * Verify that there are no ordered extents for a given file range.
10942  * we have flushed all delalloc in the range, we have waited for all ordered
10949 	struct btrfs_ordered_extent *ordered;  in btrfs_assert_inode_range_clean()  local
10954 	ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);  in btrfs_assert_inode_range_clean()
10955 	if (ordered) {  in btrfs_assert_inode_range_clean()
10957 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range…  in btrfs_assert_inode_range_clean()
10959 			  ordered->file_offset,  in btrfs_assert_inode_range_clean()
10960 			  ordered->file_offset + ordered->num_bytes - 1);  in btrfs_assert_inode_range_clean()
10961 		btrfs_put_ordered_extent(ordered);  in btrfs_assert_inode_range_clean()
10964 	ASSERT(ordered == NULL);  in btrfs_assert_inode_range_clean()