890 static void scrub_repair_read_endio(struct btrfs_bio *bbio)  in scrub_repair_read_endio()  argument
892 	struct scrub_stripe *stripe = bbio->private;  in scrub_repair_read_endio()
894 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));  in scrub_repair_read_endio()
895 	const u32 bio_size = bio_get_size(&bbio->bio);  in scrub_repair_read_endio()
899 	if (bbio->bio.bi_status) {  in scrub_repair_read_endio()
908 	bio_put(&bbio->bio);  in scrub_repair_read_endio()
919 static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe,  in scrub_bio_add_sector()  argument
922 	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;  in scrub_bio_add_sector()
926 	ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), fs_info->sectorsize,  in scrub_bio_add_sector()
944 	struct btrfs_bio *bbio;  in alloc_scrub_bbio()  local
946 	bbio = btrfs_bio_alloc(nr_vecs, opf, BTRFS_I(fs_info->btree_inode),  in alloc_scrub_bbio()
948 	bbio->is_scrub = true;  in alloc_scrub_bbio()
949 	bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT;  in alloc_scrub_bbio()
950 	return bbio;  in alloc_scrub_bbio()
957 	struct btrfs_bio *bbio = NULL;  in scrub_stripe_submit_repair_read()  local
967 		if (bbio && ((i > 0 && !test_bit(i - 1, &old_error_bitmap)) ||  in scrub_stripe_submit_repair_read()
968 			     bbio->bio.bi_iter.bi_size >= blocksize)) {  in scrub_stripe_submit_repair_read()
969 			ASSERT(bbio->bio.bi_iter.bi_size);  in scrub_stripe_submit_repair_read()
971 			btrfs_submit_bbio(bbio, mirror);  in scrub_stripe_submit_repair_read()
974 			bbio = NULL;  in scrub_stripe_submit_repair_read()
977 		if (!bbio)  in scrub_stripe_submit_repair_read()
978 			bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ,  in scrub_stripe_submit_repair_read()
982 		scrub_bio_add_sector(bbio, stripe, i);  in scrub_stripe_submit_repair_read()
984 	if (bbio) {  in scrub_stripe_submit_repair_read()
985 		ASSERT(bbio->bio.bi_iter.bi_size);  in scrub_stripe_submit_repair_read()
987 		btrfs_submit_bbio(bbio, mirror);  in scrub_stripe_submit_repair_read()
1244 static void scrub_read_endio(struct btrfs_bio *bbio)  in scrub_read_endio()  argument
1246 	struct scrub_stripe *stripe = bbio->private;  in scrub_read_endio()
1247 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));  in scrub_read_endio()
1249 	const u32 bio_size = bio_get_size(&bbio->bio);  in scrub_read_endio()
1254 	if (bbio->bio.bi_status) {  in scrub_read_endio()
1260 	bio_put(&bbio->bio);  in scrub_read_endio()
1268 static void scrub_write_endio(struct btrfs_bio *bbio)  in scrub_write_endio()  argument
1270 	struct scrub_stripe *stripe = bbio->private;  in scrub_write_endio()
1272 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));  in scrub_write_endio()
1273 	const u32 bio_size = bio_get_size(&bbio->bio);  in scrub_write_endio()
1275 	if (bbio->bio.bi_status) {  in scrub_write_endio()
1286 	bio_put(&bbio->bio);  in scrub_write_endio()
1294 				   struct btrfs_bio *bbio, bool dev_replace)  in scrub_submit_write_bio()  argument
1297 	u32 bio_len = bbio->bio.bi_iter.bi_size;  in scrub_submit_write_bio()
1298 	u32 bio_off = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT) -  in scrub_submit_write_bio()
1303 	btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace);  in scrub_submit_write_bio()
1338 	struct btrfs_bio *bbio = NULL;  in scrub_write_sectors()  local
1346 		if (bbio && sector_nr && !test_bit(sector_nr - 1, &write_bitmap)) {  in scrub_write_sectors()
1347 			scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);  in scrub_write_sectors()
1348 			bbio = NULL;  in scrub_write_sectors()
1350 		if (!bbio)  in scrub_write_sectors()
1351 			bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_WRITE,  in scrub_write_sectors()
1354 		scrub_bio_add_sector(bbio, stripe, sector_nr);  in scrub_write_sectors()
1356 	if (bbio)  in scrub_write_sectors()
1357 		scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);  in scrub_write_sectors()
1790 	struct btrfs_bio *bbio = NULL;  in scrub_submit_extent_sector_read()  local
1805 		if (bbio &&  in scrub_submit_extent_sector_read()
1807 		     bbio->bio.bi_iter.bi_size >= stripe_len)) {  in scrub_submit_extent_sector_read()
1808 			ASSERT(bbio->bio.bi_iter.bi_size);  in scrub_submit_extent_sector_read()
1810 			btrfs_submit_bbio(bbio, mirror);  in scrub_submit_extent_sector_read()
1811 			bbio = NULL;  in scrub_submit_extent_sector_read()
1814 		if (!bbio) {  in scrub_submit_extent_sector_read()
1846 			bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ,  in scrub_submit_extent_sector_read()
1850 		scrub_bio_add_sector(bbio, stripe, i);  in scrub_submit_extent_sector_read()
1853 	if (bbio) {  in scrub_submit_extent_sector_read()
1854 		ASSERT(bbio->bio.bi_iter.bi_size);  in scrub_submit_extent_sector_read()
1856 		btrfs_submit_bbio(bbio, mirror);  in scrub_submit_extent_sector_read()
1870 	struct btrfs_bio *bbio;  in scrub_submit_initial_read()  local
1884 	bbio = alloc_scrub_bbio(fs_info, BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ,  in scrub_submit_initial_read()
1888 		scrub_bio_add_sector(bbio, stripe, cur);  in scrub_submit_initial_read()
1904 	btrfs_submit_bbio(bbio, mirror);  in scrub_submit_initial_read()