24 	struct btrfs_bio *bbio;  member
30 static inline bool is_data_bbio(const struct btrfs_bio *bbio)  in is_data_bbio()  argument
32 	return bbio->inode && is_data_inode(bbio->inode);  in is_data_bbio()
35 static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio)  in bbio_has_ordered_extent()  argument
37 	return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;  in bbio_has_ordered_extent()
44 void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset,  in btrfs_bio_init()  argument
50 	memset(bbio, 0, offsetof(struct btrfs_bio, bio));  in btrfs_bio_init()
51 	bbio->inode = inode;  in btrfs_bio_init()
52 	bbio->end_io = end_io;  in btrfs_bio_init()
53 	bbio->private = private;  in btrfs_bio_init()
54 	bbio->file_offset = file_offset;  in btrfs_bio_init()
55 	atomic_set(&bbio->pending_ios, 1);  in btrfs_bio_init()
56 	WRITE_ONCE(bbio->status, BLK_STS_OK);  in btrfs_bio_init()
70 	struct btrfs_bio *bbio;  in btrfs_bio_alloc()  local
74 	bbio = btrfs_bio(bio);  in btrfs_bio_alloc()
75 	btrfs_bio_init(bbio, inode, file_offset, end_io, private);  in btrfs_bio_alloc()
76 	return bbio;  in btrfs_bio_alloc()
83 	struct btrfs_bio *bbio;  in btrfs_split_bio()  local
91 	bbio = btrfs_bio(bio);  in btrfs_split_bio()
92 	btrfs_bio_init(bbio, orig_bbio->inode, orig_bbio->file_offset, NULL, orig_bbio);  in btrfs_split_bio()
94 	if (bbio_has_ordered_extent(bbio)) {  in btrfs_split_bio()
96 		bbio->ordered = orig_bbio->ordered;  in btrfs_split_bio()
97 		bbio->orig_logical = orig_bbio->orig_logical;  in btrfs_split_bio()
101 	bbio->csum_search_commit_root = orig_bbio->csum_search_commit_root;  in btrfs_split_bio()
102 	bbio->can_use_append = orig_bbio->can_use_append;  in btrfs_split_bio()
103 	bbio->is_scrub = orig_bbio->is_scrub;  in btrfs_split_bio()
104 	bbio->is_remap = orig_bbio->is_remap;  in btrfs_split_bio()
105 	bbio->async_csum = orig_bbio->async_csum;  in btrfs_split_bio()
108 	return bbio;  in btrfs_split_bio()
111 void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status)  in btrfs_bio_end_io()  argument
116 	if (bbio->async_csum)  in btrfs_bio_end_io()
117 		wait_for_completion(&bbio->csum_done);  in btrfs_bio_end_io()
119 	bbio->bio.bi_status = status;  in btrfs_bio_end_io()
120 	if (bbio->bio.bi_pool == &btrfs_clone_bioset) {  in btrfs_bio_end_io()
121 		struct btrfs_bio *orig_bbio = bbio->private;  in btrfs_bio_end_io()
124 		if (bbio_has_ordered_extent(bbio))  in btrfs_bio_end_io()
125 			btrfs_put_ordered_extent(bbio->ordered);  in btrfs_bio_end_io()
126 		bio_put(&bbio->bio);  in btrfs_bio_end_io()
128 		bbio = orig_bbio;  in btrfs_bio_end_io()
136 		cmpxchg(&bbio->status, BLK_STS_OK, status);  in btrfs_bio_end_io()
138 	if (atomic_dec_and_test(&bbio->pending_ios)) {  in btrfs_bio_end_io()
141 			bbio->bio.bi_status = READ_ONCE(bbio->status);  in btrfs_bio_end_io()
143 		if (bbio_has_ordered_extent(bbio)) {  in btrfs_bio_end_io()
144 			struct btrfs_ordered_extent *ordered = bbio->ordered;  in btrfs_bio_end_io()
146 			bbio->end_io(bbio);  in btrfs_bio_end_io()
149 			bbio->end_io(bbio);  in btrfs_bio_end_io()
171 		btrfs_bio_end_io(fbio->bbio, fbio->bbio->bio.bi_status);  in btrfs_repair_done()
210 		if (mirror == fbio->bbio->mirror_num) {  in btrfs_end_repair_bio()
212 			fbio->bbio->bio.bi_status = BLK_STS_IOERR;  in btrfs_end_repair_bio()
225 	} while (mirror != fbio->bbio->mirror_num);  in btrfs_end_repair_bio()
272 		fbio->bbio = failed_bbio;  in repair_one_sector()
302 static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *dev)  in btrfs_check_read_bio()  argument
304 	struct btrfs_inode *inode = bbio->inode;  in btrfs_check_read_bio()
309 	struct bvec_iter *iter = &bbio->saved_iter;  in btrfs_check_read_bio()
310 	blk_status_t status = bbio->bio.bi_status;  in btrfs_check_read_bio()
323 	if (bbio->bio.bi_pool == &btrfs_repair_bioset) {  in btrfs_check_read_bio()
324 		btrfs_end_repair_bio(bbio, dev);  in btrfs_check_read_bio()
329 	bbio->bio.bi_status = BLK_STS_OK;  in btrfs_check_read_bio()
331 	btrfs_bio_for_each_block(paddr, &bbio->bio, iter, step) {  in btrfs_check_read_bio()
337 			    !btrfs_data_csum_ok(bbio, dev, offset - sectorsize, paddrs))  in btrfs_check_read_bio()
338 				fbio = repair_one_sector(bbio, offset - sectorsize,  in btrfs_check_read_bio()
342 	if (bbio->csum != bbio->csum_inline)  in btrfs_check_read_bio()
343 		kvfree(bbio->csum);  in btrfs_check_read_bio()
348 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_check_read_bio()
376 	struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work);  in simple_end_io_work()  local
377 	struct bio *bio = &bbio->bio;  in simple_end_io_work()
381 		if (is_data_bbio(bbio))  in simple_end_io_work()
382 			return btrfs_check_read_bio(bbio, bbio->bio.bi_private);  in simple_end_io_work()
383 		return btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in simple_end_io_work()
386 		btrfs_record_physical_zoned(bbio);  in simple_end_io_work()
387 	btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in simple_end_io_work()
392 	struct btrfs_bio *bbio = btrfs_bio(bio);  in btrfs_simple_end_io()  local
394 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in btrfs_simple_end_io()
401 	INIT_WORK(&bbio->end_io_work, simple_end_io_work);  in btrfs_simple_end_io()
402 	queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work);  in btrfs_simple_end_io()
408 	struct btrfs_bio *bbio = btrfs_bio(bio);  in btrfs_raid56_end_io()  local
414 	bbio->mirror_num = bioc->mirror_num;  in btrfs_raid56_end_io()
415 	if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio))  in btrfs_raid56_end_io()
416 		btrfs_check_read_bio(bbio, NULL);  in btrfs_raid56_end_io()
418 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_raid56_end_io()
425 	struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work);  in orig_write_end_io_work()  local
426 	struct bio *bio = &bbio->bio;  in orig_write_end_io_work()
449 	btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in orig_write_end_io_work()
455 	struct btrfs_bio *bbio = btrfs_bio(bio);  in btrfs_orig_write_end_io()  local
457 	INIT_WORK(&bbio->end_io_work, orig_write_end_io_work);  in btrfs_orig_write_end_io()
458 	queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work);  in btrfs_orig_write_end_io()
463 	struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work);  in clone_write_end_io_work()  local
464 	struct bio *bio = &bbio->bio;  in clone_write_end_io_work()
481 	struct btrfs_bio *bbio = btrfs_bio(bio);  in btrfs_clone_write_end_io()  local
483 	INIT_WORK(&bbio->end_io_work, clone_write_end_io_work);  in btrfs_clone_write_end_io()
484 	queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work);  in btrfs_clone_write_end_io()
590 static int btrfs_bio_csum(struct btrfs_bio *bbio)  in btrfs_bio_csum()  argument
592 	if (bbio->bio.bi_opf & REQ_META)  in btrfs_bio_csum()
593 		return btree_csum_one_bio(bbio);  in btrfs_bio_csum()
595 	return btrfs_csum_one_bio(bbio, true);  in btrfs_bio_csum()
597 	return btrfs_csum_one_bio(bbio, false);  in btrfs_bio_csum()
606 	struct btrfs_bio *bbio;  member
627 	ret = btrfs_bio_csum(async->bbio);  in run_one_async_start()
629 		async->bbio->bio.bi_status = errno_to_blk_status(ret);  in run_one_async_start()
646 	struct bio *bio = &async->bbio->bio;  in run_one_async_done()
655 		btrfs_bio_end_io(async->bbio, bio->bi_status);  in run_one_async_done()
668 static bool should_async_write(struct btrfs_bio *bbio)  in should_async_write()  argument
670 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in should_async_write()
690 	if (op_is_sync(bbio->bio.bi_opf))  in should_async_write()
694 	if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(fs_info))  in should_async_write()
705 static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio,  in btrfs_wq_submit_bio()  argument
709 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in btrfs_wq_submit_bio()
716 	async->bbio = bbio;  in btrfs_wq_submit_bio()
726 static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length)  in btrfs_append_map_length()  argument
728 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in btrfs_append_map_length()
733 	sector_offset = bio_split_rw_at(&bbio->bio, &fs_info->limits,  in btrfs_append_map_length()
746 static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)  in btrfs_submit_chunk()  argument
748 	struct btrfs_inode *inode = bbio->inode;  in btrfs_submit_chunk()
750 	struct bio *bio = &bbio->bio;  in btrfs_submit_chunk()
759 	if (bbio->is_scrub || btrfs_is_data_reloc_root(inode->root))  in btrfs_submit_chunk()
778 	if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio))  in btrfs_submit_chunk()
779 		bbio->orig_logical = logical;  in btrfs_submit_chunk()
781 	bbio->can_use_append = btrfs_use_zone_append(bbio);  in btrfs_submit_chunk()
784 	if (bbio->can_use_append)  in btrfs_submit_chunk()
785 		map_length = btrfs_append_map_length(bbio, map_length);  in btrfs_submit_chunk()
790 		split = btrfs_split_bio(fs_info, bbio, map_length);  in btrfs_submit_chunk()
796 		bbio = split;  in btrfs_submit_chunk()
797 		bio = &bbio->bio;  in btrfs_submit_chunk()
804 	if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) {  in btrfs_submit_chunk()
805 		bbio->saved_iter = bio->bi_iter;  in btrfs_submit_chunk()
806 		ret = btrfs_lookup_bio_sums(bbio);  in btrfs_submit_chunk()
813 		if (is_data_bbio(bbio) && bioc && bioc->use_rst) {  in btrfs_submit_chunk()
821 			list_add_tail(&bioc->rst_ordered_entry, &bbio->ordered->bioc_list);  in btrfs_submit_chunk()
830 		    !btrfs_is_data_reloc_root(inode->root) && !bbio->is_remap) {  in btrfs_submit_chunk()
831 			if (should_async_write(bbio) &&  in btrfs_submit_chunk()
832 			    btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num))  in btrfs_submit_chunk()
835 			ret = btrfs_bio_csum(bbio);  in btrfs_submit_chunk()
839 		} else if (bbio->can_use_append ||  in btrfs_submit_chunk()
841 			ret = btrfs_alloc_dummy_sum(bbio);  in btrfs_submit_chunk()
859 		struct btrfs_bio *remaining = bbio->private;  in btrfs_submit_chunk()
861 		ASSERT(bbio->bio.bi_pool == &btrfs_clone_bioset);  in btrfs_submit_chunk()
867 	btrfs_bio_end_io(bbio, status);  in btrfs_submit_chunk()
872 static void assert_bbio_alignment(struct btrfs_bio *bbio)  in assert_bbio_alignment()  argument
875 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in assert_bbio_alignment()
880 	const u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;  in assert_bbio_alignment()
881 	const u32 length = bbio->bio.bi_iter.bi_size;  in assert_bbio_alignment()
886 	       btrfs_root_id(bbio->inode->root),  in assert_bbio_alignment()
887 	       btrfs_ino(bbio->inode), logical, length);  in assert_bbio_alignment()
889 	bio_for_each_bvec(bvec, &bbio->bio, iter)  in assert_bbio_alignment()
893 		btrfs_root_id(bbio->inode->root),  in assert_bbio_alignment()
894 		btrfs_ino(bbio->inode), logical, length, iter.bi_idx,  in assert_bbio_alignment()
899 void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num)  in btrfs_submit_bbio()  argument
902 	ASSERT(bbio->inode || bbio->file_offset == 0);  in btrfs_submit_bbio()
904 	assert_bbio_alignment(bbio);  in btrfs_submit_bbio()
906 	while (!btrfs_submit_chunk(bbio, mirror_num))  in btrfs_submit_bbio()
1005 void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace)  in btrfs_submit_repair_write()  argument
1007 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in btrfs_submit_repair_write()
1008 	u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_repair_write()
1009 	u64 length = bbio->bio.bi_iter.bi_size;  in btrfs_submit_repair_write()
1014 	ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);  in btrfs_submit_repair_write()
1015 	ASSERT(!is_data_inode(bbio->inode));  in btrfs_submit_repair_write()
1016 	ASSERT(bbio->is_scrub);  in btrfs_submit_repair_write()
1027 	btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num);  in btrfs_submit_repair_write()
1032 	btrfs_bio_end_io(bbio, errno_to_blk_status(ret));  in btrfs_submit_repair_write()