Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0
13 #include "disk-io.h"
14 #include "ordered-data.h"
18 #include "dev-replace.h"
19 #include "check-integrity.h"
20 #include "rcu-string.h"
22 #include "block-group.h"
25 * This is only the first step towards a full-features scrub. It reads all
27 * is found or the extent cannot be read, good data will be written back if
31 * - In case an unrepairable extent is encountered, track which files are
33 * - track and record media errors, throw out bad devices
34 * - add a mode to also read unallocated space
115 /* The following is for the data used to check parity */
116 /* It is for the data with checksum */
143 /* Mark the parity blocks which have data */
147 * Mark the parity blocks which have data, but errors happen when
148 * read data or check data
185 * Use a ref counter to avoid use-after-free issues. Scrub workers
261 return page->recover && in scrub_is_page_on_raid56()
262 (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK); in scrub_is_page_on_raid56()
267 refcount_inc(&sctx->refs); in scrub_pending_bio_inc()
268 atomic_inc(&sctx->bios_in_flight); in scrub_pending_bio_inc()
273 atomic_dec(&sctx->bios_in_flight); in scrub_pending_bio_dec()
274 wake_up(&sctx->list_wait); in scrub_pending_bio_dec()
280 while (atomic_read(&fs_info->scrub_pause_req)) { in __scrub_blocked_if_needed()
281 mutex_unlock(&fs_info->scrub_lock); in __scrub_blocked_if_needed()
282 wait_event(fs_info->scrub_pause_wait, in __scrub_blocked_if_needed()
283 atomic_read(&fs_info->scrub_pause_req) == 0); in __scrub_blocked_if_needed()
284 mutex_lock(&fs_info->scrub_lock); in __scrub_blocked_if_needed()
290 atomic_inc(&fs_info->scrubs_paused); in scrub_pause_on()
291 wake_up(&fs_info->scrub_pause_wait); in scrub_pause_on()
296 mutex_lock(&fs_info->scrub_lock); in scrub_pause_off()
298 atomic_dec(&fs_info->scrubs_paused); in scrub_pause_off()
299 mutex_unlock(&fs_info->scrub_lock); in scrub_pause_off()
301 wake_up(&fs_info->scrub_pause_wait); in scrub_pause_off()
315 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
317 * NOTE: caller must hold full_stripe_locks_root->lock before calling this
329 lockdep_assert_held(&locks_root->lock); in insert_full_stripe_lock()
331 p = &locks_root->root.rb_node; in insert_full_stripe_lock()
335 if (fstripe_logical < entry->logical) { in insert_full_stripe_lock()
336 p = &(*p)->rb_left; in insert_full_stripe_lock()
337 } else if (fstripe_logical > entry->logical) { in insert_full_stripe_lock()
338 p = &(*p)->rb_right; in insert_full_stripe_lock()
340 entry->refs++; in insert_full_stripe_lock()
350 return ERR_PTR(-ENOMEM); in insert_full_stripe_lock()
351 ret->logical = fstripe_logical; in insert_full_stripe_lock()
352 ret->refs = 1; in insert_full_stripe_lock()
353 mutex_init(&ret->mutex); in insert_full_stripe_lock()
355 rb_link_node(&ret->node, parent, p); in insert_full_stripe_lock()
356 rb_insert_color(&ret->node, &locks_root->root); in insert_full_stripe_lock()
373 lockdep_assert_held(&locks_root->lock); in search_full_stripe_lock()
375 node = locks_root->root.rb_node; in search_full_stripe_lock()
378 if (fstripe_logical < entry->logical) in search_full_stripe_lock()
379 node = node->rb_left; in search_full_stripe_lock()
380 else if (fstripe_logical > entry->logical) in search_full_stripe_lock()
381 node = node->rb_right; in search_full_stripe_lock()
401 WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX); in get_full_stripe_logical()
407 ret = div64_u64(bytenr - cache->start, cache->full_stripe_len) * in get_full_stripe_logical()
408 cache->full_stripe_len + cache->start; in get_full_stripe_logical()
436 return -ENOENT; in lock_full_stripe()
440 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) in lock_full_stripe()
442 locks_root = &bg_cache->full_stripe_locks_root; in lock_full_stripe()
447 mutex_lock(&locks_root->lock); in lock_full_stripe()
449 mutex_unlock(&locks_root->lock); in lock_full_stripe()
454 mutex_lock(&existing->mutex); in lock_full_stripe()
487 return -ENOENT; in unlock_full_stripe()
489 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) in unlock_full_stripe()
492 locks_root = &bg_cache->full_stripe_locks_root; in unlock_full_stripe()
495 mutex_lock(&locks_root->lock); in unlock_full_stripe()
500 ret = -ENOENT; in unlock_full_stripe()
501 mutex_unlock(&locks_root->lock); in unlock_full_stripe()
505 if (fstripe_lock->refs == 0) { in unlock_full_stripe()
508 fstripe_lock->logical); in unlock_full_stripe()
510 fstripe_lock->refs--; in unlock_full_stripe()
513 if (fstripe_lock->refs == 0) { in unlock_full_stripe()
514 rb_erase(&fstripe_lock->node, &locks_root->root); in unlock_full_stripe()
517 mutex_unlock(&locks_root->lock); in unlock_full_stripe()
519 mutex_unlock(&fstripe_lock->mutex); in unlock_full_stripe()
529 while (!list_empty(&sctx->csum_list)) { in scrub_free_csums()
531 sum = list_first_entry(&sctx->csum_list, in scrub_free_csums()
533 list_del(&sum->list); in scrub_free_csums()
546 if (sctx->curr != -1) { in scrub_free_ctx()
547 struct scrub_bio *sbio = sctx->bios[sctx->curr]; in scrub_free_ctx()
549 for (i = 0; i < sbio->page_count; i++) { in scrub_free_ctx()
550 WARN_ON(!sbio->pagev[i]->page); in scrub_free_ctx()
551 scrub_block_put(sbio->pagev[i]->sblock); in scrub_free_ctx()
553 bio_put(sbio->bio); in scrub_free_ctx()
557 struct scrub_bio *sbio = sctx->bios[i]; in scrub_free_ctx()
564 kfree(sctx->wr_curr_bio); in scrub_free_ctx()
571 if (refcount_dec_and_test(&sctx->refs)) in scrub_put_ctx()
584 refcount_set(&sctx->refs, 1); in scrub_setup_ctx()
585 sctx->is_dev_replace = is_dev_replace; in scrub_setup_ctx()
586 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; in scrub_setup_ctx()
587 sctx->curr = -1; in scrub_setup_ctx()
588 sctx->fs_info = fs_info; in scrub_setup_ctx()
589 INIT_LIST_HEAD(&sctx->csum_list); in scrub_setup_ctx()
596 sctx->bios[i] = sbio; in scrub_setup_ctx()
598 sbio->index = i; in scrub_setup_ctx()
599 sbio->sctx = sctx; in scrub_setup_ctx()
600 sbio->page_count = 0; in scrub_setup_ctx()
601 btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL, in scrub_setup_ctx()
604 if (i != SCRUB_BIOS_PER_SCTX - 1) in scrub_setup_ctx()
605 sctx->bios[i]->next_free = i + 1; in scrub_setup_ctx()
607 sctx->bios[i]->next_free = -1; in scrub_setup_ctx()
609 sctx->first_free = 0; in scrub_setup_ctx()
610 atomic_set(&sctx->bios_in_flight, 0); in scrub_setup_ctx()
611 atomic_set(&sctx->workers_pending, 0); in scrub_setup_ctx()
612 atomic_set(&sctx->cancel_req, 0); in scrub_setup_ctx()
613 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy); in scrub_setup_ctx()
615 spin_lock_init(&sctx->list_lock); in scrub_setup_ctx()
616 spin_lock_init(&sctx->stat_lock); in scrub_setup_ctx()
617 init_waitqueue_head(&sctx->list_wait); in scrub_setup_ctx()
619 WARN_ON(sctx->wr_curr_bio != NULL); in scrub_setup_ctx()
620 mutex_init(&sctx->wr_lock); in scrub_setup_ctx()
621 sctx->wr_curr_bio = NULL; in scrub_setup_ctx()
623 WARN_ON(!fs_info->dev_replace.tgtdev); in scrub_setup_ctx()
624 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; in scrub_setup_ctx()
625 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; in scrub_setup_ctx()
626 sctx->flush_all_writes = false; in scrub_setup_ctx()
633 return ERR_PTR(-ENOMEM); in scrub_setup_ctx()
647 struct btrfs_fs_info *fs_info = swarn->dev->fs_info; in scrub_print_warning_inode()
665 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0); in scrub_print_warning_inode()
668 btrfs_release_path(swarn->path); in scrub_print_warning_inode()
672 eb = swarn->path->nodes[0]; in scrub_print_warning_inode()
673 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], in scrub_print_warning_inode()
677 btrfs_release_path(swarn->path); in scrub_print_warning_inode()
685 ipath = init_ipath(4096, local_root, swarn->path); in scrub_print_warning_inode()
702 for (i = 0; i < ipath->fspath->elem_cnt; ++i) in scrub_print_warning_inode()
705 swarn->errstr, swarn->logical, in scrub_print_warning_inode()
706 rcu_str_deref(swarn->dev->name), in scrub_print_warning_inode()
707 swarn->physical, in scrub_print_warning_inode()
709 min(isize - offset, (u64)PAGE_SIZE), nlink, in scrub_print_warning_inode()
710 (char *)(unsigned long)ipath->fspath->val[i]); in scrub_print_warning_inode()
719 swarn->errstr, swarn->logical, in scrub_print_warning_inode()
720 rcu_str_deref(swarn->dev->name), in scrub_print_warning_inode()
721 swarn->physical, in scrub_print_warning_inode()
745 WARN_ON(sblock->page_count < 1); in scrub_print_warning()
746 dev = sblock->pagev[0]->dev; in scrub_print_warning()
747 fs_info = sblock->sctx->fs_info; in scrub_print_warning()
753 swarn.physical = sblock->pagev[0]->physical; in scrub_print_warning()
754 swarn.logical = sblock->pagev[0]->logical; in scrub_print_warning()
763 extent_item_pos = swarn.logical - found_key.objectid; in scrub_print_warning()
766 eb = path->nodes[0]; in scrub_print_warning()
767 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); in scrub_print_warning()
768 item_size = btrfs_item_size_nr(eb, path->slots[0]); in scrub_print_warning()
778 rcu_str_deref(dev->name), in scrub_print_warning()
781 ret < 0 ? -1 : ref_level, in scrub_print_warning()
782 ret < 0 ? -1 : ref_root); in scrub_print_warning()
800 refcount_inc(&recover->refs); in scrub_get_recover()
806 if (refcount_dec_and_test(&recover->refs)) { in scrub_put_recover()
808 btrfs_put_bbio(recover->bbio); in scrub_put_recover()
823 struct scrub_ctx *sctx = sblock_to_check->sctx; in scrub_handle_errored_block()
830 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */ in scrub_handle_errored_block()
841 BUG_ON(sblock_to_check->page_count < 1); in scrub_handle_errored_block()
842 fs_info = sctx->fs_info; in scrub_handle_errored_block()
843 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { in scrub_handle_errored_block()
849 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
850 ++sctx->stat.super_errors; in scrub_handle_errored_block()
851 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
854 logical = sblock_to_check->pagev[0]->logical; in scrub_handle_errored_block()
855 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1); in scrub_handle_errored_block()
856 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1; in scrub_handle_errored_block()
857 is_metadata = !(sblock_to_check->pagev[0]->flags & in scrub_handle_errored_block()
859 have_csum = sblock_to_check->pagev[0]->have_csum; in scrub_handle_errored_block()
860 dev = sblock_to_check->pagev[0]->dev; in scrub_handle_errored_block()
874 * For data corruption, Parity and Data threads will both try in scrub_handle_errored_block()
875 * to recovery the data. in scrub_handle_errored_block()
882 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
883 if (ret == -ENOMEM) in scrub_handle_errored_block()
884 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
885 sctx->stat.read_errors++; in scrub_handle_errored_block()
886 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
887 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
893 * re-read the extent or metadata block that failed (that was in scrub_handle_errored_block()
898 * mirror contains I/O errors, but the errors do not in scrub_handle_errored_block()
899 * overlap, i.e. the data can be repaired by selecting the in scrub_handle_errored_block()
902 * would be that mirror #1 has an I/O error on the first page, in scrub_handle_errored_block()
903 * the second page is good, and mirror #2 has an I/O error on in scrub_handle_errored_block()
905 * Then the first page of the first mirror can be repaired by in scrub_handle_errored_block()
906 * taking the first page of the second mirror, and the in scrub_handle_errored_block()
907 * second page of the second mirror can be repaired by in scrub_handle_errored_block()
908 * copying the contents of the 2nd page of the 1st mirror. in scrub_handle_errored_block()
909 * One more note: if the pages of one mirror contain I/O in scrub_handle_errored_block()
911 * the best data for repairing, the first attempt is to find in scrub_handle_errored_block()
912 * a mirror without I/O errors and with a validated checksum. in scrub_handle_errored_block()
923 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
924 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
925 sctx->stat.read_errors++; in scrub_handle_errored_block()
926 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
927 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
935 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
936 sctx->stat.read_errors++; in scrub_handle_errored_block()
937 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
938 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
945 /* build and submit the bios for the failed mirror, check checksums */ in scrub_handle_errored_block()
948 if (!sblock_bad->header_error && !sblock_bad->checksum_error && in scrub_handle_errored_block()
949 sblock_bad->no_io_error_seen) { in scrub_handle_errored_block()
958 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
959 sctx->stat.unverified_errors++; in scrub_handle_errored_block()
960 sblock_to_check->data_corrected = 1; in scrub_handle_errored_block()
961 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
963 if (sctx->is_dev_replace) in scrub_handle_errored_block()
968 if (!sblock_bad->no_io_error_seen) { in scrub_handle_errored_block()
969 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
970 sctx->stat.read_errors++; in scrub_handle_errored_block()
971 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
975 } else if (sblock_bad->checksum_error) { in scrub_handle_errored_block()
976 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
977 sctx->stat.csum_errors++; in scrub_handle_errored_block()
978 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
983 } else if (sblock_bad->header_error) { in scrub_handle_errored_block()
984 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
985 sctx->stat.verify_errors++; in scrub_handle_errored_block()
986 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
990 if (sblock_bad->generation_error) in scrub_handle_errored_block()
998 if (sctx->readonly) { in scrub_handle_errored_block()
999 ASSERT(!sctx->is_dev_replace); in scrub_handle_errored_block()
1006 * First try to pick the mirror which is completely without I/O in scrub_handle_errored_block()
1011 * If a mirror is found which is completely correct, and no in scrub_handle_errored_block()
1024 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */ in scrub_handle_errored_block()
1025 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) { in scrub_handle_errored_block()
1033 struct scrub_recover *r = sblock_bad->pagev[0]->recover; in scrub_handle_errored_block()
1034 int max_allowed = r->bbio->num_stripes - in scrub_handle_errored_block()
1035 r->bbio->num_tgtdevs; in scrub_handle_errored_block()
1044 sblock_other->pagev[0]->mirror_num = 1 + mirror_index; in scrub_handle_errored_block()
1050 if (!sblock_other->header_error && in scrub_handle_errored_block()
1051 !sblock_other->checksum_error && in scrub_handle_errored_block()
1052 sblock_other->no_io_error_seen) { in scrub_handle_errored_block()
1053 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1065 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace) in scrub_handle_errored_block()
1079 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page in scrub_handle_errored_block()
1080 * of mirror #2 is readable but the final checksum test fails, in scrub_handle_errored_block()
1081 * then the 2nd page of mirror #3 could be tried, whether now in scrub_handle_errored_block()
1088 * mirror could be repaired by taking 512 byte of a different in scrub_handle_errored_block()
1089 * mirror, even if other 512 byte sectors in the same PAGE_SIZE in scrub_handle_errored_block()
1093 for (page_num = 0; page_num < sblock_bad->page_count; in scrub_handle_errored_block()
1095 struct scrub_page *page_bad = sblock_bad->pagev[page_num]; in scrub_handle_errored_block()
1098 /* skip no-io-error page in scrub */ in scrub_handle_errored_block()
1099 if (!page_bad->io_error && !sctx->is_dev_replace) in scrub_handle_errored_block()
1102 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) { in scrub_handle_errored_block()
1105 * didn't work out correct data, then copy the content in scrub_handle_errored_block()
1107 * to source device, instead of writing garbage data in in scrub_handle_errored_block()
1111 } else if (page_bad->io_error) { in scrub_handle_errored_block()
1112 /* try to find no-io-error page in mirrors */ in scrub_handle_errored_block()
1118 pagev[page_num]->io_error) { in scrub_handle_errored_block()
1128 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1130 * did not find a mirror to fetch the page in scrub_handle_errored_block()
1132 * handles this case (page->io_error), by in scrub_handle_errored_block()
1142 &fs_info->dev_replace.num_write_errors); in scrub_handle_errored_block()
1150 page_bad->io_error = 0; in scrub_handle_errored_block()
1156 if (success && !sctx->is_dev_replace) { in scrub_handle_errored_block()
1161 * request for data to be repaired is on its way). in scrub_handle_errored_block()
1163 * which re-reads the data before the checksum in scrub_handle_errored_block()
1164 * is verified, but most likely the data comes out in scrub_handle_errored_block()
1168 if (!sblock_bad->header_error && in scrub_handle_errored_block()
1169 !sblock_bad->checksum_error && in scrub_handle_errored_block()
1170 sblock_bad->no_io_error_seen) in scrub_handle_errored_block()
1176 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1177 sctx->stat.corrected_errors++; in scrub_handle_errored_block()
1178 sblock_to_check->data_corrected = 1; in scrub_handle_errored_block()
1179 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1182 logical, rcu_str_deref(dev->name)); in scrub_handle_errored_block()
1186 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1187 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
1188 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1191 logical, rcu_str_deref(dev->name)); in scrub_handle_errored_block()
1203 for (page_index = 0; page_index < sblock->page_count; in scrub_handle_errored_block()
1205 sblock->pagev[page_index]->sblock = NULL; in scrub_handle_errored_block()
1206 recover = sblock->pagev[page_index]->recover; in scrub_handle_errored_block()
1209 sblock->pagev[page_index]->recover = in scrub_handle_errored_block()
1212 scrub_page_put(sblock->pagev[page_index]); in scrub_handle_errored_block()
1227 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5) in scrub_nr_raid_mirrors()
1229 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) in scrub_nr_raid_mirrors()
1232 return (int)bbio->num_stripes; in scrub_nr_raid_mirrors()
1238 int nstripes, int mirror, in scrub_stripe_index_and_offset() argument
1257 *stripe_offset = logical - raid_map[i]; in scrub_stripe_index_and_offset()
1260 *stripe_index = mirror; in scrub_stripe_index_and_offset()
1268 struct scrub_ctx *sctx = original_sblock->sctx; in scrub_setup_recheck_block()
1269 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_setup_recheck_block()
1270 u64 length = original_sblock->page_count * PAGE_SIZE; in scrub_setup_recheck_block()
1271 u64 logical = original_sblock->pagev[0]->logical; in scrub_setup_recheck_block()
1272 u64 generation = original_sblock->pagev[0]->generation; in scrub_setup_recheck_block()
1273 u64 flags = original_sblock->pagev[0]->flags; in scrub_setup_recheck_block()
1274 u64 have_csum = original_sblock->pagev[0]->have_csum; in scrub_setup_recheck_block()
1299 * represents one mirror in scrub_setup_recheck_block()
1307 return -EIO; in scrub_setup_recheck_block()
1314 return -ENOMEM; in scrub_setup_recheck_block()
1317 refcount_set(&recover->refs, 1); in scrub_setup_recheck_block()
1318 recover->bbio = bbio; in scrub_setup_recheck_block()
1319 recover->map_length = mapped_length; in scrub_setup_recheck_block()
1331 sblock->sctx = sctx; in scrub_setup_recheck_block()
1336 spin_lock(&sctx->stat_lock); in scrub_setup_recheck_block()
1337 sctx->stat.malloc_errors++; in scrub_setup_recheck_block()
1338 spin_unlock(&sctx->stat_lock); in scrub_setup_recheck_block()
1340 return -ENOMEM; in scrub_setup_recheck_block()
1343 sblock->pagev[page_index] = page; in scrub_setup_recheck_block()
1344 page->sblock = sblock; in scrub_setup_recheck_block()
1345 page->flags = flags; in scrub_setup_recheck_block()
1346 page->generation = generation; in scrub_setup_recheck_block()
1347 page->logical = logical; in scrub_setup_recheck_block()
1348 page->have_csum = have_csum; in scrub_setup_recheck_block()
1350 memcpy(page->csum, in scrub_setup_recheck_block()
1351 original_sblock->pagev[0]->csum, in scrub_setup_recheck_block()
1352 sctx->csum_size); in scrub_setup_recheck_block()
1355 bbio->map_type, in scrub_setup_recheck_block()
1356 bbio->raid_map, in scrub_setup_recheck_block()
1358 bbio->num_stripes - in scrub_setup_recheck_block()
1359 bbio->num_tgtdevs, in scrub_setup_recheck_block()
1363 page->physical = bbio->stripes[stripe_index].physical + in scrub_setup_recheck_block()
1365 page->dev = bbio->stripes[stripe_index].dev; in scrub_setup_recheck_block()
1367 BUG_ON(page_index >= original_sblock->page_count); in scrub_setup_recheck_block()
1368 page->physical_for_dev_replace = in scrub_setup_recheck_block()
1369 original_sblock->pagev[page_index]-> in scrub_setup_recheck_block()
1371 /* for missing devices, dev->bdev is NULL */ in scrub_setup_recheck_block()
1372 page->mirror_num = mirror_index + 1; in scrub_setup_recheck_block()
1373 sblock->page_count++; in scrub_setup_recheck_block()
1374 page->page = alloc_page(GFP_NOFS); in scrub_setup_recheck_block()
1375 if (!page->page) in scrub_setup_recheck_block()
1379 page->recover = recover; in scrub_setup_recheck_block()
1382 length -= sublen; in scrub_setup_recheck_block()
1392 complete(bio->bi_private); in scrub_bio_wait_endio()
1403 bio->bi_iter.bi_sector = page->logical >> 9; in scrub_submit_raid56_bio_wait()
1404 bio->bi_private = &done; in scrub_submit_raid56_bio_wait()
1405 bio->bi_end_io = scrub_bio_wait_endio; in scrub_submit_raid56_bio_wait()
1407 mirror_num = page->sblock->pagev[0]->mirror_num; in scrub_submit_raid56_bio_wait()
1408 ret = raid56_parity_recover(fs_info, bio, page->recover->bbio, in scrub_submit_raid56_bio_wait()
1409 page->recover->map_length, in scrub_submit_raid56_bio_wait()
1415 return blk_status_to_errno(bio->bi_status); in scrub_submit_raid56_bio_wait()
1421 struct scrub_page *first_page = sblock->pagev[0]; in scrub_recheck_block_on_raid56()
1426 ASSERT(first_page->dev); in scrub_recheck_block_on_raid56()
1427 if (!first_page->dev->bdev) in scrub_recheck_block_on_raid56()
1431 bio_set_dev(bio, first_page->dev->bdev); in scrub_recheck_block_on_raid56()
1433 for (page_num = 0; page_num < sblock->page_count; page_num++) { in scrub_recheck_block_on_raid56()
1434 struct scrub_page *page = sblock->pagev[page_num]; in scrub_recheck_block_on_raid56()
1436 WARN_ON(!page->page); in scrub_recheck_block_on_raid56()
1437 bio_add_page(bio, page->page, PAGE_SIZE, 0); in scrub_recheck_block_on_raid56()
1451 for (page_num = 0; page_num < sblock->page_count; page_num++) in scrub_recheck_block_on_raid56()
1452 sblock->pagev[page_num]->io_error = 1; in scrub_recheck_block_on_raid56()
1454 sblock->no_io_error_seen = 0; in scrub_recheck_block_on_raid56()
1458 * this function will check the on disk data for checksum errors, header
1462 * the pages that are errored in the just handled mirror can be repaired.
1470 sblock->no_io_error_seen = 1; in scrub_recheck_block()
1473 if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0])) in scrub_recheck_block()
1476 for (page_num = 0; page_num < sblock->page_count; page_num++) { in scrub_recheck_block()
1478 struct scrub_page *page = sblock->pagev[page_num]; in scrub_recheck_block()
1480 if (page->dev->bdev == NULL) { in scrub_recheck_block()
1481 page->io_error = 1; in scrub_recheck_block()
1482 sblock->no_io_error_seen = 0; in scrub_recheck_block()
1486 WARN_ON(!page->page); in scrub_recheck_block()
1488 bio_set_dev(bio, page->dev->bdev); in scrub_recheck_block()
1490 bio_add_page(bio, page->page, PAGE_SIZE, 0); in scrub_recheck_block()
1491 bio->bi_iter.bi_sector = page->physical >> 9; in scrub_recheck_block()
1492 bio->bi_opf = REQ_OP_READ; in scrub_recheck_block()
1495 page->io_error = 1; in scrub_recheck_block()
1496 sblock->no_io_error_seen = 0; in scrub_recheck_block()
1502 if (sblock->no_io_error_seen) in scrub_recheck_block()
1509 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices; in scrub_check_fsid()
1512 ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE); in scrub_check_fsid()
1518 sblock->header_error = 0; in scrub_recheck_block_checksum()
1519 sblock->checksum_error = 0; in scrub_recheck_block_checksum()
1520 sblock->generation_error = 0; in scrub_recheck_block_checksum()
1522 if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA) in scrub_recheck_block_checksum()
1534 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) { in scrub_repair_block_from_good_copy()
1551 struct scrub_page *page_bad = sblock_bad->pagev[page_num]; in scrub_repair_page_from_good_copy()
1552 struct scrub_page *page_good = sblock_good->pagev[page_num]; in scrub_repair_page_from_good_copy()
1553 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; in scrub_repair_page_from_good_copy()
1555 BUG_ON(page_bad->page == NULL); in scrub_repair_page_from_good_copy()
1556 BUG_ON(page_good->page == NULL); in scrub_repair_page_from_good_copy()
1557 if (force_write || sblock_bad->header_error || in scrub_repair_page_from_good_copy()
1558 sblock_bad->checksum_error || page_bad->io_error) { in scrub_repair_page_from_good_copy()
1562 if (!page_bad->dev->bdev) { in scrub_repair_page_from_good_copy()
1565 return -EIO; in scrub_repair_page_from_good_copy()
1569 bio_set_dev(bio, page_bad->dev->bdev); in scrub_repair_page_from_good_copy()
1570 bio->bi_iter.bi_sector = page_bad->physical >> 9; in scrub_repair_page_from_good_copy()
1571 bio->bi_opf = REQ_OP_WRITE; in scrub_repair_page_from_good_copy()
1573 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0); in scrub_repair_page_from_good_copy()
1576 return -EIO; in scrub_repair_page_from_good_copy()
1580 btrfs_dev_stat_inc_and_print(page_bad->dev, in scrub_repair_page_from_good_copy()
1582 atomic64_inc(&fs_info->dev_replace.num_write_errors); in scrub_repair_page_from_good_copy()
1584 return -EIO; in scrub_repair_page_from_good_copy()
1594 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_write_block_to_dev_replace()
1599 * so the data needn't be written into the destination device. in scrub_write_block_to_dev_replace()
1601 if (sblock->sparity) in scrub_write_block_to_dev_replace()
1604 for (page_num = 0; page_num < sblock->page_count; page_num++) { in scrub_write_block_to_dev_replace()
1609 atomic64_inc(&fs_info->dev_replace.num_write_errors); in scrub_write_block_to_dev_replace()
1616 struct scrub_page *spage = sblock->pagev[page_num]; in scrub_write_page_to_dev_replace()
1618 BUG_ON(spage->page == NULL); in scrub_write_page_to_dev_replace()
1619 if (spage->io_error) in scrub_write_page_to_dev_replace()
1620 clear_page(page_address(spage->page)); in scrub_write_page_to_dev_replace()
1622 return scrub_add_page_to_wr_bio(sblock->sctx, spage); in scrub_write_page_to_dev_replace()
1631 mutex_lock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1633 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1634 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio), in scrub_add_page_to_wr_bio()
1636 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1637 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1638 return -ENOMEM; in scrub_add_page_to_wr_bio()
1640 sctx->wr_curr_bio->sctx = sctx; in scrub_add_page_to_wr_bio()
1641 sctx->wr_curr_bio->page_count = 0; in scrub_add_page_to_wr_bio()
1643 sbio = sctx->wr_curr_bio; in scrub_add_page_to_wr_bio()
1644 if (sbio->page_count == 0) { in scrub_add_page_to_wr_bio()
1647 sbio->physical = spage->physical_for_dev_replace; in scrub_add_page_to_wr_bio()
1648 sbio->logical = spage->logical; in scrub_add_page_to_wr_bio()
1649 sbio->dev = sctx->wr_tgtdev; in scrub_add_page_to_wr_bio()
1650 bio = sbio->bio; in scrub_add_page_to_wr_bio()
1652 bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio); in scrub_add_page_to_wr_bio()
1653 sbio->bio = bio; in scrub_add_page_to_wr_bio()
1656 bio->bi_private = sbio; in scrub_add_page_to_wr_bio()
1657 bio->bi_end_io = scrub_wr_bio_end_io; in scrub_add_page_to_wr_bio()
1658 bio_set_dev(bio, sbio->dev->bdev); in scrub_add_page_to_wr_bio()
1659 bio->bi_iter.bi_sector = sbio->physical >> 9; in scrub_add_page_to_wr_bio()
1660 bio->bi_opf = REQ_OP_WRITE; in scrub_add_page_to_wr_bio()
1661 sbio->status = 0; in scrub_add_page_to_wr_bio()
1662 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != in scrub_add_page_to_wr_bio()
1663 spage->physical_for_dev_replace || in scrub_add_page_to_wr_bio()
1664 sbio->logical + sbio->page_count * PAGE_SIZE != in scrub_add_page_to_wr_bio()
1665 spage->logical) { in scrub_add_page_to_wr_bio()
1670 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); in scrub_add_page_to_wr_bio()
1672 if (sbio->page_count < 1) { in scrub_add_page_to_wr_bio()
1673 bio_put(sbio->bio); in scrub_add_page_to_wr_bio()
1674 sbio->bio = NULL; in scrub_add_page_to_wr_bio()
1675 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1676 return -EIO; in scrub_add_page_to_wr_bio()
1682 sbio->pagev[sbio->page_count] = spage; in scrub_add_page_to_wr_bio()
1684 sbio->page_count++; in scrub_add_page_to_wr_bio()
1685 if (sbio->page_count == sctx->pages_per_wr_bio) in scrub_add_page_to_wr_bio()
1687 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1696 if (!sctx->wr_curr_bio) in scrub_wr_submit()
1699 sbio = sctx->wr_curr_bio; in scrub_wr_submit()
1700 sctx->wr_curr_bio = NULL; in scrub_wr_submit()
1701 WARN_ON(!sbio->bio->bi_disk); in scrub_wr_submit()
1707 btrfsic_submit_bio(sbio->bio); in scrub_wr_submit()
1712 struct scrub_bio *sbio = bio->bi_private; in scrub_wr_bio_end_io()
1713 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; in scrub_wr_bio_end_io()
1715 sbio->status = bio->bi_status; in scrub_wr_bio_end_io()
1716 sbio->bio = bio; in scrub_wr_bio_end_io()
1718 btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL); in scrub_wr_bio_end_io()
1719 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work); in scrub_wr_bio_end_io()
1725 struct scrub_ctx *sctx = sbio->sctx; in scrub_wr_bio_end_io_worker()
1728 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO); in scrub_wr_bio_end_io_worker()
1729 if (sbio->status) { in scrub_wr_bio_end_io_worker()
1731 &sbio->sctx->fs_info->dev_replace; in scrub_wr_bio_end_io_worker()
1733 for (i = 0; i < sbio->page_count; i++) { in scrub_wr_bio_end_io_worker()
1734 struct scrub_page *spage = sbio->pagev[i]; in scrub_wr_bio_end_io_worker()
1736 spage->io_error = 1; in scrub_wr_bio_end_io_worker()
1737 atomic64_inc(&dev_replace->num_write_errors); in scrub_wr_bio_end_io_worker()
1741 for (i = 0; i < sbio->page_count; i++) in scrub_wr_bio_end_io_worker()
1742 scrub_page_put(sbio->pagev[i]); in scrub_wr_bio_end_io_worker()
1744 bio_put(sbio->bio); in scrub_wr_bio_end_io_worker()
1762 sblock->header_error = 0; in scrub_checksum()
1763 sblock->generation_error = 0; in scrub_checksum()
1764 sblock->checksum_error = 0; in scrub_checksum()
1766 WARN_ON(sblock->page_count < 1); in scrub_checksum()
1767 flags = sblock->pagev[0]->flags; in scrub_checksum()
1785 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_data()
1786 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_data()
1787 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); in scrub_checksum_data()
1792 BUG_ON(sblock->page_count < 1); in scrub_checksum_data()
1793 spage = sblock->pagev[0]; in scrub_checksum_data()
1794 if (!spage->have_csum) in scrub_checksum_data()
1797 kaddr = page_address(spage->page); in scrub_checksum_data()
1799 shash->tfm = fs_info->csum_shash; in scrub_checksum_data()
1803 if (memcmp(csum, spage->csum, sctx->csum_size)) in scrub_checksum_data()
1804 sblock->checksum_error = 1; in scrub_checksum_data()
1806 return sblock->checksum_error; in scrub_checksum_data()
1811 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_tree_block()
1813 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_tree_block()
1814 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); in scrub_checksum_tree_block()
1817 const int num_pages = sctx->fs_info->nodesize >> PAGE_SHIFT; in scrub_checksum_tree_block()
1822 BUG_ON(sblock->page_count < 1); in scrub_checksum_tree_block()
1823 spage = sblock->pagev[0]; in scrub_checksum_tree_block()
1824 kaddr = page_address(spage->page); in scrub_checksum_tree_block()
1826 memcpy(on_disk_csum, h->csum, sctx->csum_size); in scrub_checksum_tree_block()
1833 if (spage->logical != btrfs_stack_header_bytenr(h)) in scrub_checksum_tree_block()
1834 sblock->header_error = 1; in scrub_checksum_tree_block()
1836 if (spage->generation != btrfs_stack_header_generation(h)) { in scrub_checksum_tree_block()
1837 sblock->header_error = 1; in scrub_checksum_tree_block()
1838 sblock->generation_error = 1; in scrub_checksum_tree_block()
1841 if (!scrub_check_fsid(h->fsid, spage)) in scrub_checksum_tree_block()
1842 sblock->header_error = 1; in scrub_checksum_tree_block()
1844 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid, in scrub_checksum_tree_block()
1846 sblock->header_error = 1; in scrub_checksum_tree_block()
1848 shash->tfm = fs_info->csum_shash; in scrub_checksum_tree_block()
1851 PAGE_SIZE - BTRFS_CSUM_SIZE); in scrub_checksum_tree_block()
1854 kaddr = page_address(sblock->pagev[i]->page); in scrub_checksum_tree_block()
1859 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) in scrub_checksum_tree_block()
1860 sblock->checksum_error = 1; in scrub_checksum_tree_block()
1862 return sblock->header_error || sblock->checksum_error; in scrub_checksum_tree_block()
1868 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_super()
1869 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_super()
1870 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); in scrub_checksum_super()
1877 BUG_ON(sblock->page_count < 1); in scrub_checksum_super()
1878 spage = sblock->pagev[0]; in scrub_checksum_super()
1879 kaddr = page_address(spage->page); in scrub_checksum_super()
1882 if (spage->logical != btrfs_super_bytenr(s)) in scrub_checksum_super()
1885 if (spage->generation != btrfs_super_generation(s)) in scrub_checksum_super()
1888 if (!scrub_check_fsid(s->fsid, spage)) in scrub_checksum_super()
1891 shash->tfm = fs_info->csum_shash; in scrub_checksum_super()
1894 BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, calculated_csum); in scrub_checksum_super()
1896 if (memcmp(calculated_csum, s->csum, sctx->csum_size)) in scrub_checksum_super()
1905 spin_lock(&sctx->stat_lock); in scrub_checksum_super()
1906 ++sctx->stat.super_errors; in scrub_checksum_super()
1907 spin_unlock(&sctx->stat_lock); in scrub_checksum_super()
1909 btrfs_dev_stat_inc_and_print(spage->dev, in scrub_checksum_super()
1912 btrfs_dev_stat_inc_and_print(spage->dev, in scrub_checksum_super()
1921 refcount_inc(&sblock->refs); in scrub_block_get()
1926 if (refcount_dec_and_test(&sblock->refs)) { in scrub_block_put()
1929 if (sblock->sparity) in scrub_block_put()
1930 scrub_parity_put(sblock->sparity); in scrub_block_put()
1932 for (i = 0; i < sblock->page_count; i++) in scrub_block_put()
1933 scrub_page_put(sblock->pagev[i]); in scrub_block_put()
1940 atomic_inc(&spage->refs); in scrub_page_get()
1945 if (atomic_dec_and_test(&spage->refs)) { in scrub_page_put()
1946 if (spage->page) in scrub_page_put()
1947 __free_page(spage->page); in scrub_page_put()
1956 if (sctx->curr == -1) in scrub_submit()
1959 sbio = sctx->bios[sctx->curr]; in scrub_submit()
1960 sctx->curr = -1; in scrub_submit()
1962 btrfsic_submit_bio(sbio->bio); in scrub_submit()
1968 struct scrub_block *sblock = spage->sblock; in scrub_add_page_to_rd_bio()
1976 while (sctx->curr == -1) { in scrub_add_page_to_rd_bio()
1977 spin_lock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
1978 sctx->curr = sctx->first_free; in scrub_add_page_to_rd_bio()
1979 if (sctx->curr != -1) { in scrub_add_page_to_rd_bio()
1980 sctx->first_free = sctx->bios[sctx->curr]->next_free; in scrub_add_page_to_rd_bio()
1981 sctx->bios[sctx->curr]->next_free = -1; in scrub_add_page_to_rd_bio()
1982 sctx->bios[sctx->curr]->page_count = 0; in scrub_add_page_to_rd_bio()
1983 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
1985 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
1986 wait_event(sctx->list_wait, sctx->first_free != -1); in scrub_add_page_to_rd_bio()
1989 sbio = sctx->bios[sctx->curr]; in scrub_add_page_to_rd_bio()
1990 if (sbio->page_count == 0) { in scrub_add_page_to_rd_bio()
1993 sbio->physical = spage->physical; in scrub_add_page_to_rd_bio()
1994 sbio->logical = spage->logical; in scrub_add_page_to_rd_bio()
1995 sbio->dev = spage->dev; in scrub_add_page_to_rd_bio()
1996 bio = sbio->bio; in scrub_add_page_to_rd_bio()
1998 bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio); in scrub_add_page_to_rd_bio()
1999 sbio->bio = bio; in scrub_add_page_to_rd_bio()
2002 bio->bi_private = sbio; in scrub_add_page_to_rd_bio()
2003 bio->bi_end_io = scrub_bio_end_io; in scrub_add_page_to_rd_bio()
2004 bio_set_dev(bio, sbio->dev->bdev); in scrub_add_page_to_rd_bio()
2005 bio->bi_iter.bi_sector = sbio->physical >> 9; in scrub_add_page_to_rd_bio()
2006 bio->bi_opf = REQ_OP_READ; in scrub_add_page_to_rd_bio()
2007 sbio->status = 0; in scrub_add_page_to_rd_bio()
2008 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != in scrub_add_page_to_rd_bio()
2009 spage->physical || in scrub_add_page_to_rd_bio()
2010 sbio->logical + sbio->page_count * PAGE_SIZE != in scrub_add_page_to_rd_bio()
2011 spage->logical || in scrub_add_page_to_rd_bio()
2012 sbio->dev != spage->dev) { in scrub_add_page_to_rd_bio()
2017 sbio->pagev[sbio->page_count] = spage; in scrub_add_page_to_rd_bio()
2018 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); in scrub_add_page_to_rd_bio()
2020 if (sbio->page_count < 1) { in scrub_add_page_to_rd_bio()
2021 bio_put(sbio->bio); in scrub_add_page_to_rd_bio()
2022 sbio->bio = NULL; in scrub_add_page_to_rd_bio()
2023 return -EIO; in scrub_add_page_to_rd_bio()
2030 atomic_inc(&sblock->outstanding_pages); in scrub_add_page_to_rd_bio()
2031 sbio->page_count++; in scrub_add_page_to_rd_bio()
2032 if (sbio->page_count == sctx->pages_per_rd_bio) in scrub_add_page_to_rd_bio()
2040 struct scrub_block *sblock = bio->bi_private; in scrub_missing_raid56_end_io()
2041 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_missing_raid56_end_io()
2043 if (bio->bi_status) in scrub_missing_raid56_end_io()
2044 sblock->no_io_error_seen = 0; in scrub_missing_raid56_end_io()
2048 btrfs_queue_work(fs_info->scrub_workers, &sblock->work); in scrub_missing_raid56_end_io()
2054 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_worker()
2055 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_worker()
2059 logical = sblock->pagev[0]->logical; in scrub_missing_raid56_worker()
2060 dev = sblock->pagev[0]->dev; in scrub_missing_raid56_worker()
2062 if (sblock->no_io_error_seen) in scrub_missing_raid56_worker()
2065 if (!sblock->no_io_error_seen) { in scrub_missing_raid56_worker()
2066 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2067 sctx->stat.read_errors++; in scrub_missing_raid56_worker()
2068 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2071 logical, rcu_str_deref(dev->name)); in scrub_missing_raid56_worker()
2072 } else if (sblock->header_error || sblock->checksum_error) { in scrub_missing_raid56_worker()
2073 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2074 sctx->stat.uncorrectable_errors++; in scrub_missing_raid56_worker()
2075 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2078 logical, rcu_str_deref(dev->name)); in scrub_missing_raid56_worker()
2083 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_missing_raid56_worker()
2084 mutex_lock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2086 mutex_unlock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2095 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_pages()
2096 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_pages()
2097 u64 length = sblock->page_count * PAGE_SIZE; in scrub_missing_raid56_pages()
2098 u64 logical = sblock->pagev[0]->logical; in scrub_missing_raid56_pages()
2108 if (ret || !bbio || !bbio->raid_map) in scrub_missing_raid56_pages()
2111 if (WARN_ON(!sctx->is_dev_replace || in scrub_missing_raid56_pages()
2112 !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) { in scrub_missing_raid56_pages()
2123 bio->bi_iter.bi_sector = logical >> 9; in scrub_missing_raid56_pages()
2124 bio->bi_private = sblock; in scrub_missing_raid56_pages()
2125 bio->bi_end_io = scrub_missing_raid56_end_io; in scrub_missing_raid56_pages()
2131 for (i = 0; i < sblock->page_count; i++) { in scrub_missing_raid56_pages()
2132 struct scrub_page *spage = sblock->pagev[i]; in scrub_missing_raid56_pages()
2134 raid56_add_scrub_pages(rbio, spage->page, spage->logical); in scrub_missing_raid56_pages()
2137 btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL); in scrub_missing_raid56_pages()
2148 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2149 sctx->stat.malloc_errors++; in scrub_missing_raid56_pages()
2150 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2163 spin_lock(&sctx->stat_lock); in scrub_pages()
2164 sctx->stat.malloc_errors++; in scrub_pages()
2165 spin_unlock(&sctx->stat_lock); in scrub_pages()
2166 return -ENOMEM; in scrub_pages()
2171 refcount_set(&sblock->refs, 1); in scrub_pages()
2172 sblock->sctx = sctx; in scrub_pages()
2173 sblock->no_io_error_seen = 1; in scrub_pages()
2182 spin_lock(&sctx->stat_lock); in scrub_pages()
2183 sctx->stat.malloc_errors++; in scrub_pages()
2184 spin_unlock(&sctx->stat_lock); in scrub_pages()
2186 return -ENOMEM; in scrub_pages()
2190 sblock->pagev[index] = spage; in scrub_pages()
2191 spage->sblock = sblock; in scrub_pages()
2192 spage->dev = dev; in scrub_pages()
2193 spage->flags = flags; in scrub_pages()
2194 spage->generation = gen; in scrub_pages()
2195 spage->logical = logical; in scrub_pages()
2196 spage->physical = physical; in scrub_pages()
2197 spage->physical_for_dev_replace = physical_for_dev_replace; in scrub_pages()
2198 spage->mirror_num = mirror_num; in scrub_pages()
2200 spage->have_csum = 1; in scrub_pages()
2201 memcpy(spage->csum, csum, sctx->csum_size); in scrub_pages()
2203 spage->have_csum = 0; in scrub_pages()
2205 sblock->page_count++; in scrub_pages()
2206 spage->page = alloc_page(GFP_KERNEL); in scrub_pages()
2207 if (!spage->page) in scrub_pages()
2209 len -= l; in scrub_pages()
2215 WARN_ON(sblock->page_count == 0); in scrub_pages()
2216 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) { in scrub_pages()
2223 for (index = 0; index < sblock->page_count; index++) { in scrub_pages()
2224 struct scrub_page *spage = sblock->pagev[index]; in scrub_pages()
2245 struct scrub_bio *sbio = bio->bi_private; in scrub_bio_end_io()
2246 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; in scrub_bio_end_io()
2248 sbio->status = bio->bi_status; in scrub_bio_end_io()
2249 sbio->bio = bio; in scrub_bio_end_io()
2251 btrfs_queue_work(fs_info->scrub_workers, &sbio->work); in scrub_bio_end_io()
2257 struct scrub_ctx *sctx = sbio->sctx; in scrub_bio_end_io_worker()
2260 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO); in scrub_bio_end_io_worker()
2261 if (sbio->status) { in scrub_bio_end_io_worker()
2262 for (i = 0; i < sbio->page_count; i++) { in scrub_bio_end_io_worker()
2263 struct scrub_page *spage = sbio->pagev[i]; in scrub_bio_end_io_worker()
2265 spage->io_error = 1; in scrub_bio_end_io_worker()
2266 spage->sblock->no_io_error_seen = 0; in scrub_bio_end_io_worker()
2271 for (i = 0; i < sbio->page_count; i++) { in scrub_bio_end_io_worker()
2272 struct scrub_page *spage = sbio->pagev[i]; in scrub_bio_end_io_worker()
2273 struct scrub_block *sblock = spage->sblock; in scrub_bio_end_io_worker()
2275 if (atomic_dec_and_test(&sblock->outstanding_pages)) in scrub_bio_end_io_worker()
2280 bio_put(sbio->bio); in scrub_bio_end_io_worker()
2281 sbio->bio = NULL; in scrub_bio_end_io_worker()
2282 spin_lock(&sctx->list_lock); in scrub_bio_end_io_worker()
2283 sbio->next_free = sctx->first_free; in scrub_bio_end_io_worker()
2284 sctx->first_free = sbio->index; in scrub_bio_end_io_worker()
2285 spin_unlock(&sctx->list_lock); in scrub_bio_end_io_worker()
2287 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_bio_end_io_worker()
2288 mutex_lock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2290 mutex_unlock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2303 int sectorsize = sparity->sctx->fs_info->sectorsize; in __scrub_mark_bitmap()
2305 if (len >= sparity->stripe_len) { in __scrub_mark_bitmap()
2306 bitmap_set(bitmap, 0, sparity->nsectors); in __scrub_mark_bitmap()
2310 start -= sparity->logic_start; in __scrub_mark_bitmap()
2311 start = div64_u64_rem(start, sparity->stripe_len, &offset); in __scrub_mark_bitmap()
2318 if (offset + nsectors <= sparity->nsectors) { in __scrub_mark_bitmap()
2323 bitmap_set(bitmap, offset, sparity->nsectors - offset); in __scrub_mark_bitmap()
2324 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset)); in __scrub_mark_bitmap()
2330 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len); in scrub_parity_mark_sectors_error()
2336 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len); in scrub_parity_mark_sectors_data()
2343 if (!sblock->no_io_error_seen) { in scrub_block_complete()
2353 if (!corrupted && sblock->sctx->is_dev_replace) in scrub_block_complete()
2357 if (sblock->sparity && corrupted && !sblock->data_corrected) { in scrub_block_complete()
2358 u64 start = sblock->pagev[0]->logical; in scrub_block_complete()
2359 u64 end = sblock->pagev[sblock->page_count - 1]->logical + in scrub_block_complete()
2362 scrub_parity_mark_sectors_error(sblock->sparity, in scrub_block_complete()
2363 start, end - start); in scrub_block_complete()
2373 while (!list_empty(&sctx->csum_list)) { in scrub_find_csum()
2374 sum = list_first_entry(&sctx->csum_list, in scrub_find_csum()
2376 if (sum->bytenr > logical) in scrub_find_csum()
2378 if (sum->bytenr + sum->len > logical) in scrub_find_csum()
2381 ++sctx->stat.csum_discards; in scrub_find_csum()
2382 list_del(&sum->list); in scrub_find_csum()
2389 index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize); in scrub_find_csum()
2392 num_sectors = sum->len / sctx->fs_info->sectorsize; in scrub_find_csum()
2393 memcpy(csum, sum->sums + index * sctx->csum_size, sctx->csum_size); in scrub_find_csum()
2394 if (index == num_sectors - 1) { in scrub_find_csum()
2395 list_del(&sum->list); in scrub_find_csum()
2412 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) in scrub_extent()
2413 blocksize = map->stripe_len; in scrub_extent()
2415 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2416 spin_lock(&sctx->stat_lock); in scrub_extent()
2417 sctx->stat.data_extents_scrubbed++; in scrub_extent()
2418 sctx->stat.data_bytes_scrubbed += len; in scrub_extent()
2419 spin_unlock(&sctx->stat_lock); in scrub_extent()
2421 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) in scrub_extent()
2422 blocksize = map->stripe_len; in scrub_extent()
2424 blocksize = sctx->fs_info->nodesize; in scrub_extent()
2425 spin_lock(&sctx->stat_lock); in scrub_extent()
2426 sctx->stat.tree_extents_scrubbed++; in scrub_extent()
2427 sctx->stat.tree_bytes_scrubbed += len; in scrub_extent()
2428 spin_unlock(&sctx->stat_lock); in scrub_extent()
2430 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2442 ++sctx->stat.no_csum; in scrub_extent()
2449 len -= l; in scrub_extent()
2462 struct scrub_ctx *sctx = sparity->sctx; in scrub_pages_for_parity()
2468 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2469 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2470 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2471 return -ENOMEM; in scrub_pages_for_parity()
2476 refcount_set(&sblock->refs, 1); in scrub_pages_for_parity()
2477 sblock->sctx = sctx; in scrub_pages_for_parity()
2478 sblock->no_io_error_seen = 1; in scrub_pages_for_parity()
2479 sblock->sparity = sparity; in scrub_pages_for_parity()
2489 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2490 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2491 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2493 return -ENOMEM; in scrub_pages_for_parity()
2498 sblock->pagev[index] = spage; in scrub_pages_for_parity()
2501 list_add_tail(&spage->list, &sparity->spages); in scrub_pages_for_parity()
2502 spage->sblock = sblock; in scrub_pages_for_parity()
2503 spage->dev = dev; in scrub_pages_for_parity()
2504 spage->flags = flags; in scrub_pages_for_parity()
2505 spage->generation = gen; in scrub_pages_for_parity()
2506 spage->logical = logical; in scrub_pages_for_parity()
2507 spage->physical = physical; in scrub_pages_for_parity()
2508 spage->mirror_num = mirror_num; in scrub_pages_for_parity()
2510 spage->have_csum = 1; in scrub_pages_for_parity()
2511 memcpy(spage->csum, csum, sctx->csum_size); in scrub_pages_for_parity()
2513 spage->have_csum = 0; in scrub_pages_for_parity()
2515 sblock->page_count++; in scrub_pages_for_parity()
2516 spage->page = alloc_page(GFP_KERNEL); in scrub_pages_for_parity()
2517 if (!spage->page) in scrub_pages_for_parity()
2519 len -= l; in scrub_pages_for_parity()
2524 WARN_ON(sblock->page_count == 0); in scrub_pages_for_parity()
2525 for (index = 0; index < sblock->page_count; index++) { in scrub_pages_for_parity()
2526 struct scrub_page *spage = sblock->pagev[index]; in scrub_pages_for_parity()
2546 struct scrub_ctx *sctx = sparity->sctx; in scrub_extent_for_parity()
2551 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) { in scrub_extent_for_parity()
2557 blocksize = sparity->stripe_len; in scrub_extent_for_parity()
2559 blocksize = sparity->stripe_len; in scrub_extent_for_parity()
2561 blocksize = sctx->fs_info->sectorsize; in scrub_extent_for_parity()
2581 len -= l; in scrub_extent_for_parity()
2591 * the most left data stripe's logical offset.
2593 * return 0 if it is a data stripe, 1 means parity stripe.
2607 last_offset = (physical - map->stripes[num].physical) * data_stripes; in get_raid56_logic_offset()
2613 *offset = last_offset + i * map->stripe_len; in get_raid56_logic_offset()
2615 stripe_nr = div64_u64(*offset, map->stripe_len); in get_raid56_logic_offset()
2618 /* Work out the disk rotation on this stripe-set */ in get_raid56_logic_offset()
2619 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot); in get_raid56_logic_offset()
2620 /* calculate which stripe this data locates */ in get_raid56_logic_offset()
2622 stripe_index = rot % map->num_stripes; in get_raid56_logic_offset()
2628 *offset = last_offset + j * map->stripe_len; in get_raid56_logic_offset()
2634 struct scrub_ctx *sctx = sparity->sctx; in scrub_free_parity()
2638 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors); in scrub_free_parity()
2640 spin_lock(&sctx->stat_lock); in scrub_free_parity()
2641 sctx->stat.read_errors += nbits; in scrub_free_parity()
2642 sctx->stat.uncorrectable_errors += nbits; in scrub_free_parity()
2643 spin_unlock(&sctx->stat_lock); in scrub_free_parity()
2646 list_for_each_entry_safe(curr, next, &sparity->spages, list) { in scrub_free_parity()
2647 list_del_init(&curr->list); in scrub_free_parity()
2658 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_bio_endio_worker()
2666 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; in scrub_parity_bio_endio()
2667 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info; in scrub_parity_bio_endio()
2669 if (bio->bi_status) in scrub_parity_bio_endio()
2670 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, in scrub_parity_bio_endio()
2671 sparity->nsectors); in scrub_parity_bio_endio()
2675 btrfs_init_work(&sparity->work, scrub_parity_bio_endio_worker, NULL, in scrub_parity_bio_endio()
2677 btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work); in scrub_parity_bio_endio()
2682 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_check_and_repair()
2683 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_parity_check_and_repair()
2690 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap, in scrub_parity_check_and_repair()
2691 sparity->nsectors)) in scrub_parity_check_and_repair()
2694 length = sparity->logic_end - sparity->logic_start; in scrub_parity_check_and_repair()
2697 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start, in scrub_parity_check_and_repair()
2699 if (ret || !bbio || !bbio->raid_map) in scrub_parity_check_and_repair()
2703 bio->bi_iter.bi_sector = sparity->logic_start >> 9; in scrub_parity_check_and_repair()
2704 bio->bi_private = sparity; in scrub_parity_check_and_repair()
2705 bio->bi_end_io = scrub_parity_bio_endio; in scrub_parity_check_and_repair()
2708 length, sparity->scrub_dev, in scrub_parity_check_and_repair()
2709 sparity->dbitmap, in scrub_parity_check_and_repair()
2710 sparity->nsectors); in scrub_parity_check_and_repair()
2723 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, in scrub_parity_check_and_repair()
2724 sparity->nsectors); in scrub_parity_check_and_repair()
2725 spin_lock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2726 sctx->stat.malloc_errors++; in scrub_parity_check_and_repair()
2727 spin_unlock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2739 refcount_inc(&sparity->refs); in scrub_parity_get()
2744 if (!refcount_dec_and_test(&sparity->refs)) in scrub_parity_put()
2757 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_raid56_parity()
2758 struct btrfs_root *root = fs_info->extent_root; in scrub_raid56_parity()
2759 struct btrfs_root *csum_root = fs_info->csum_root; in scrub_raid56_parity()
2779 nsectors = div_u64(map->stripe_len, fs_info->sectorsize); in scrub_raid56_parity()
2784 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
2785 sctx->stat.malloc_errors++; in scrub_raid56_parity()
2786 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
2787 return -ENOMEM; in scrub_raid56_parity()
2790 sparity->stripe_len = map->stripe_len; in scrub_raid56_parity()
2791 sparity->nsectors = nsectors; in scrub_raid56_parity()
2792 sparity->sctx = sctx; in scrub_raid56_parity()
2793 sparity->scrub_dev = sdev; in scrub_raid56_parity()
2794 sparity->logic_start = logic_start; in scrub_raid56_parity()
2795 sparity->logic_end = logic_end; in scrub_raid56_parity()
2796 refcount_set(&sparity->refs, 1); in scrub_raid56_parity()
2797 INIT_LIST_HEAD(&sparity->spages); in scrub_raid56_parity()
2798 sparity->dbitmap = sparity->bitmap; in scrub_raid56_parity()
2799 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len; in scrub_raid56_parity()
2808 key.offset = (u64)-1; in scrub_raid56_parity()
2831 l = path->nodes[0]; in scrub_raid56_parity()
2832 slot = path->slots[0]; in scrub_raid56_parity()
2850 bytes = fs_info->nodesize; in scrub_raid56_parity()
2862 while (key.objectid >= logic_start + map->stripe_len) in scrub_raid56_parity()
2863 logic_start += map->stripe_len; in scrub_raid56_parity()
2873 logic_start + map->stripe_len)) { in scrub_raid56_parity()
2877 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
2878 sctx->stat.uncorrectable_errors++; in scrub_raid56_parity()
2879 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
2887 extent_len -= logic_start - extent_logical; in scrub_raid56_parity()
2892 logic_start + map->stripe_len) in scrub_raid56_parity()
2893 extent_len = logic_start + map->stripe_len - in scrub_raid56_parity()
2906 ret = -EIO; in scrub_raid56_parity()
2912 extent_physical = bbio->stripes[0].physical; in scrub_raid56_parity()
2913 extent_mirror_num = bbio->mirror_num; in scrub_raid56_parity()
2914 extent_dev = bbio->stripes[0].dev; in scrub_raid56_parity()
2919 extent_logical + extent_len - 1, in scrub_raid56_parity()
2920 &sctx->csum_list, 1); in scrub_raid56_parity()
2938 logic_start += map->stripe_len; in scrub_raid56_parity()
2951 path->slots[0]++; in scrub_raid56_parity()
2959 logic_start += map->stripe_len; in scrub_raid56_parity()
2964 logic_end - logic_start); in scrub_raid56_parity()
2967 mutex_lock(&sctx->wr_lock); in scrub_raid56_parity()
2969 mutex_unlock(&sctx->wr_lock); in scrub_raid56_parity()
2982 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_stripe()
2983 struct btrfs_root *root = fs_info->extent_root; in scrub_stripe()
2984 struct btrfs_root *csum_root = fs_info->csum_root; in scrub_stripe()
3002 u64 increment = map->stripe_len; in scrub_stripe()
3013 physical = map->stripes[num].physical; in scrub_stripe()
3015 nstripes = div64_u64(length, map->stripe_len); in scrub_stripe()
3016 if (map->type & BTRFS_BLOCK_GROUP_RAID0) { in scrub_stripe()
3017 offset = map->stripe_len * num; in scrub_stripe()
3018 increment = map->stripe_len * map->num_stripes; in scrub_stripe()
3020 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) { in scrub_stripe()
3021 int factor = map->num_stripes / map->sub_stripes; in scrub_stripe()
3022 offset = map->stripe_len * (num / map->sub_stripes); in scrub_stripe()
3023 increment = map->stripe_len * factor; in scrub_stripe()
3024 mirror_num = num % map->sub_stripes + 1; in scrub_stripe()
3025 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) { in scrub_stripe()
3026 increment = map->stripe_len; in scrub_stripe()
3027 mirror_num = num % map->num_stripes + 1; in scrub_stripe()
3028 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) { in scrub_stripe()
3029 increment = map->stripe_len; in scrub_stripe()
3030 mirror_num = num % map->num_stripes + 1; in scrub_stripe()
3031 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in scrub_stripe()
3033 increment = map->stripe_len * nr_data_stripes(map); in scrub_stripe()
3036 increment = map->stripe_len; in scrub_stripe()
3042 return -ENOMEM; in scrub_stripe()
3047 return -ENOMEM; in scrub_stripe()
3055 path->search_commit_root = 1; in scrub_stripe()
3056 path->skip_locking = 1; in scrub_stripe()
3058 ppath->search_commit_root = 1; in scrub_stripe()
3059 ppath->skip_locking = 1; in scrub_stripe()
3066 physical_end = physical + nstripes * map->stripe_len; in scrub_stripe()
3067 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in scrub_stripe()
3074 wait_event(sctx->list_wait, in scrub_stripe()
3075 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3084 key_end.offset = (u64)-1; in scrub_stripe()
3102 * collect all data csums for the stripe to avoid seeking during in scrub_stripe()
3115 if (atomic_read(&fs_info->scrub_cancel_req) || in scrub_stripe()
3116 atomic_read(&sctx->cancel_req)) { in scrub_stripe()
3117 ret = -ECANCELED; in scrub_stripe()
3123 if (atomic_read(&fs_info->scrub_pause_req)) { in scrub_stripe()
3125 sctx->flush_all_writes = true; in scrub_stripe()
3127 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3129 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3130 wait_event(sctx->list_wait, in scrub_stripe()
3131 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3132 sctx->flush_all_writes = false; in scrub_stripe()
3136 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in scrub_stripe()
3159 key.offset = (u64)-1; in scrub_stripe()
3184 l = path->nodes[0]; in scrub_stripe()
3185 slot = path->slots[0]; in scrub_stripe()
3203 bytes = fs_info->nodesize; in scrub_stripe()
3210 if (key.objectid >= logical + map->stripe_len) { in scrub_stripe()
3223 spin_lock(&cache->lock); in scrub_stripe()
3224 if (cache->removed) { in scrub_stripe()
3225 spin_unlock(&cache->lock); in scrub_stripe()
3229 spin_unlock(&cache->lock); in scrub_stripe()
3239 logical + map->stripe_len)) { in scrub_stripe()
3243 spin_lock(&sctx->stat_lock); in scrub_stripe()
3244 sctx->stat.uncorrectable_errors++; in scrub_stripe()
3245 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3257 extent_len -= logical - extent_logical; in scrub_stripe()
3261 logical + map->stripe_len) { in scrub_stripe()
3262 extent_len = logical + map->stripe_len - in scrub_stripe()
3266 extent_physical = extent_logical - logical + physical; in scrub_stripe()
3269 if (sctx->is_dev_replace) in scrub_stripe()
3278 extent_logical + extent_len - 1, in scrub_stripe()
3279 &sctx->csum_list, 1); in scrub_stripe()
3287 extent_logical - logical + physical); in scrub_stripe()
3296 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in scrub_stripe()
3298 * loop until we find next data stripe in scrub_stripe()
3302 physical += map->stripe_len; in scrub_stripe()
3321 physical += map->stripe_len; in scrub_stripe()
3335 path->slots[0]++; in scrub_stripe()
3340 physical += map->stripe_len; in scrub_stripe()
3341 spin_lock(&sctx->stat_lock); in scrub_stripe()
3343 sctx->stat.last_physical = map->stripes[num].physical + in scrub_stripe()
3346 sctx->stat.last_physical = physical; in scrub_stripe()
3347 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3354 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3356 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3370 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_chunk()
3371 struct extent_map_tree *map_tree = &fs_info->mapping_tree; in scrub_chunk()
3377 read_lock(&map_tree->lock); in scrub_chunk()
3379 read_unlock(&map_tree->lock); in scrub_chunk()
3386 spin_lock(&cache->lock); in scrub_chunk()
3387 if (!cache->removed) in scrub_chunk()
3388 ret = -EINVAL; in scrub_chunk()
3389 spin_unlock(&cache->lock); in scrub_chunk()
3394 map = em->map_lookup; in scrub_chunk()
3395 if (em->start != chunk_offset) in scrub_chunk()
3398 if (em->len < length) in scrub_chunk()
3401 for (i = 0; i < map->num_stripes; ++i) { in scrub_chunk()
3402 if (map->stripes[i].dev->bdev == scrub_dev->bdev && in scrub_chunk()
3403 map->stripes[i].physical == dev_offset) { in scrub_chunk()
3422 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_enumerate_chunks()
3423 struct btrfs_root *root = fs_info->dev_root; in scrub_enumerate_chunks()
3433 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; in scrub_enumerate_chunks()
3437 return -ENOMEM; in scrub_enumerate_chunks()
3439 path->reada = READA_FORWARD; in scrub_enumerate_chunks()
3440 path->search_commit_root = 1; in scrub_enumerate_chunks()
3441 path->skip_locking = 1; in scrub_enumerate_chunks()
3443 key.objectid = scrub_dev->devid; in scrub_enumerate_chunks()
3452 if (path->slots[0] >= in scrub_enumerate_chunks()
3453 btrfs_header_nritems(path->nodes[0])) { in scrub_enumerate_chunks()
3466 l = path->nodes[0]; in scrub_enumerate_chunks()
3467 slot = path->slots[0]; in scrub_enumerate_chunks()
3471 if (found_key.objectid != scrub_dev->devid) in scrub_enumerate_chunks()
3510 spin_lock(&cache->lock); in scrub_enumerate_chunks()
3511 if (cache->removed) { in scrub_enumerate_chunks()
3512 spin_unlock(&cache->lock); in scrub_enumerate_chunks()
3517 spin_unlock(&cache->lock); in scrub_enumerate_chunks()
3523 * -> btrfs_wait_for_commit() in scrub_enumerate_chunks()
3524 * -> btrfs_commit_transaction() in scrub_enumerate_chunks()
3525 * -> btrfs_scrub_pause() in scrub_enumerate_chunks()
3533 * -EFBIG from btrfs_finish_chunk_alloc() like: in scrub_enumerate_chunks()
3549 * - Write duplication in scrub_enumerate_chunks()
3550 * Contains latest data in scrub_enumerate_chunks()
3551 * - Scrub copy in scrub_enumerate_chunks()
3552 * Contains data from commit tree in scrub_enumerate_chunks()
3555 * be overwritten by scrub copy, causing data corruption. in scrub_enumerate_chunks()
3556 * So for dev-replace, it's not allowed to continue if a block in scrub_enumerate_chunks()
3559 ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); in scrub_enumerate_chunks()
3562 } else if (ret == -ENOSPC && !sctx->is_dev_replace) { in scrub_enumerate_chunks()
3564 * btrfs_inc_block_group_ro return -ENOSPC when it in scrub_enumerate_chunks()
3582 * finish before dev-replace. in scrub_enumerate_chunks()
3585 if (sctx->is_dev_replace) { in scrub_enumerate_chunks()
3587 btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, in scrub_enumerate_chunks()
3588 cache->length); in scrub_enumerate_chunks()
3592 down_write(&dev_replace->rwsem); in scrub_enumerate_chunks()
3593 dev_replace->cursor_right = found_key.offset + length; in scrub_enumerate_chunks()
3594 dev_replace->cursor_left = found_key.offset; in scrub_enumerate_chunks()
3595 dev_replace->item_needs_writeback = 1; in scrub_enumerate_chunks()
3596 up_write(&dev_replace->rwsem); in scrub_enumerate_chunks()
3611 sctx->flush_all_writes = true; in scrub_enumerate_chunks()
3613 mutex_lock(&sctx->wr_lock); in scrub_enumerate_chunks()
3615 mutex_unlock(&sctx->wr_lock); in scrub_enumerate_chunks()
3617 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3618 atomic_read(&sctx->bios_in_flight) == 0); in scrub_enumerate_chunks()
3627 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3628 atomic_read(&sctx->workers_pending) == 0); in scrub_enumerate_chunks()
3629 sctx->flush_all_writes = false; in scrub_enumerate_chunks()
3633 down_write(&dev_replace->rwsem); in scrub_enumerate_chunks()
3634 dev_replace->cursor_left = dev_replace->cursor_right; in scrub_enumerate_chunks()
3635 dev_replace->item_needs_writeback = 1; in scrub_enumerate_chunks()
3636 up_write(&dev_replace->rwsem); in scrub_enumerate_chunks()
3648 spin_lock(&cache->lock); in scrub_enumerate_chunks()
3649 if (!cache->removed && !cache->ro && cache->reserved == 0 && in scrub_enumerate_chunks()
3650 cache->used == 0) { in scrub_enumerate_chunks()
3651 spin_unlock(&cache->lock); in scrub_enumerate_chunks()
3653 btrfs_discard_queue_work(&fs_info->discard_ctl, in scrub_enumerate_chunks()
3658 spin_unlock(&cache->lock); in scrub_enumerate_chunks()
3665 if (sctx->is_dev_replace && in scrub_enumerate_chunks()
3666 atomic64_read(&dev_replace->num_write_errors) > 0) { in scrub_enumerate_chunks()
3667 ret = -EIO; in scrub_enumerate_chunks()
3670 if (sctx->stat.malloc_errors > 0) { in scrub_enumerate_chunks()
3671 ret = -ENOMEM; in scrub_enumerate_chunks()
3691 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_supers()
3693 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) in scrub_supers()
3694 return -EROFS; in scrub_supers()
3697 if (scrub_dev->fs_devices != fs_info->fs_devices) in scrub_supers()
3698 gen = scrub_dev->generation; in scrub_supers()
3700 gen = fs_info->last_trans_committed; in scrub_supers()
3705 scrub_dev->commit_total_bytes) in scrub_supers()
3714 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in scrub_supers()
3721 if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt, in scrub_workers_put()
3722 &fs_info->scrub_lock)) { in scrub_workers_put()
3727 scrub_workers = fs_info->scrub_workers; in scrub_workers_put()
3728 scrub_wr_comp = fs_info->scrub_wr_completion_workers; in scrub_workers_put()
3729 scrub_parity = fs_info->scrub_parity_workers; in scrub_workers_put()
3731 fs_info->scrub_workers = NULL; in scrub_workers_put()
3732 fs_info->scrub_wr_completion_workers = NULL; in scrub_workers_put()
3733 fs_info->scrub_parity_workers = NULL; in scrub_workers_put()
3734 mutex_unlock(&fs_info->scrub_lock); in scrub_workers_put()
3743 * get a reference count on fs_info->scrub_workers. start worker if necessary
3752 int max_active = fs_info->thread_pool_size; in scrub_workers_get()
3753 int ret = -ENOMEM; in scrub_workers_get()
3755 if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt)) in scrub_workers_get()
3773 mutex_lock(&fs_info->scrub_lock); in scrub_workers_get()
3774 if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) { in scrub_workers_get()
3775 ASSERT(fs_info->scrub_workers == NULL && in scrub_workers_get()
3776 fs_info->scrub_wr_completion_workers == NULL && in scrub_workers_get()
3777 fs_info->scrub_parity_workers == NULL); in scrub_workers_get()
3778 fs_info->scrub_workers = scrub_workers; in scrub_workers_get()
3779 fs_info->scrub_wr_completion_workers = scrub_wr_comp; in scrub_workers_get()
3780 fs_info->scrub_parity_workers = scrub_parity; in scrub_workers_get()
3781 refcount_set(&fs_info->scrub_workers_refcnt, 1); in scrub_workers_get()
3782 mutex_unlock(&fs_info->scrub_lock); in scrub_workers_get()
3786 refcount_inc(&fs_info->scrub_workers_refcnt); in scrub_workers_get()
3787 mutex_unlock(&fs_info->scrub_lock); in scrub_workers_get()
3809 return -EAGAIN; in btrfs_scrub_dev()
3811 if (fs_info->nodesize > BTRFS_STRIPE_LEN) { in btrfs_scrub_dev()
3819 fs_info->nodesize, in btrfs_scrub_dev()
3821 return -EINVAL; in btrfs_scrub_dev()
3824 if (fs_info->sectorsize != PAGE_SIZE) { in btrfs_scrub_dev()
3825 /* not supported for data w/o checksums */ in btrfs_scrub_dev()
3828 fs_info->sectorsize, PAGE_SIZE); in btrfs_scrub_dev()
3829 return -EINVAL; in btrfs_scrub_dev()
3832 if (fs_info->nodesize > in btrfs_scrub_dev()
3834 fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) { in btrfs_scrub_dev()
3841 fs_info->nodesize, in btrfs_scrub_dev()
3843 fs_info->sectorsize, in btrfs_scrub_dev()
3845 return -EINVAL; in btrfs_scrub_dev()
3857 mutex_lock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3858 dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true); in btrfs_scrub_dev()
3859 if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) && in btrfs_scrub_dev()
3861 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3862 ret = -ENODEV; in btrfs_scrub_dev()
3867 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) { in btrfs_scrub_dev()
3868 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3871 devid, rcu_str_deref(dev->name)); in btrfs_scrub_dev()
3872 ret = -EROFS; in btrfs_scrub_dev()
3876 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3877 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || in btrfs_scrub_dev()
3878 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) { in btrfs_scrub_dev()
3879 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3880 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3881 ret = -EIO; in btrfs_scrub_dev()
3885 down_read(&fs_info->dev_replace.rwsem); in btrfs_scrub_dev()
3886 if (dev->scrub_ctx || in btrfs_scrub_dev()
3888 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) { in btrfs_scrub_dev()
3889 up_read(&fs_info->dev_replace.rwsem); in btrfs_scrub_dev()
3890 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3891 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3892 ret = -EINPROGRESS; in btrfs_scrub_dev()
3895 up_read(&fs_info->dev_replace.rwsem); in btrfs_scrub_dev()
3897 sctx->readonly = readonly; in btrfs_scrub_dev()
3898 dev->scrub_ctx = sctx; in btrfs_scrub_dev()
3899 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3906 atomic_inc(&fs_info->scrubs_running); in btrfs_scrub_dev()
3907 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3916 * before incrementing fs_info->scrubs_running). in btrfs_scrub_dev()
3925 mutex_lock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3927 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_dev()
3934 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in btrfs_scrub_dev()
3935 atomic_dec(&fs_info->scrubs_running); in btrfs_scrub_dev()
3936 wake_up(&fs_info->scrub_pause_wait); in btrfs_scrub_dev()
3938 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0); in btrfs_scrub_dev()
3941 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_dev()
3947 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3948 dev->scrub_ctx = NULL; in btrfs_scrub_dev()
3949 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_dev()
3965 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_pause()
3966 atomic_inc(&fs_info->scrub_pause_req); in btrfs_scrub_pause()
3967 while (atomic_read(&fs_info->scrubs_paused) != in btrfs_scrub_pause()
3968 atomic_read(&fs_info->scrubs_running)) { in btrfs_scrub_pause()
3969 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_pause()
3970 wait_event(fs_info->scrub_pause_wait, in btrfs_scrub_pause()
3971 atomic_read(&fs_info->scrubs_paused) == in btrfs_scrub_pause()
3972 atomic_read(&fs_info->scrubs_running)); in btrfs_scrub_pause()
3973 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_pause()
3975 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_pause()
3980 atomic_dec(&fs_info->scrub_pause_req); in btrfs_scrub_continue()
3981 wake_up(&fs_info->scrub_pause_wait); in btrfs_scrub_continue()
3986 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_cancel()
3987 if (!atomic_read(&fs_info->scrubs_running)) { in btrfs_scrub_cancel()
3988 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel()
3989 return -ENOTCONN; in btrfs_scrub_cancel()
3992 atomic_inc(&fs_info->scrub_cancel_req); in btrfs_scrub_cancel()
3993 while (atomic_read(&fs_info->scrubs_running)) { in btrfs_scrub_cancel()
3994 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel()
3995 wait_event(fs_info->scrub_pause_wait, in btrfs_scrub_cancel()
3996 atomic_read(&fs_info->scrubs_running) == 0); in btrfs_scrub_cancel()
3997 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_cancel()
3999 atomic_dec(&fs_info->scrub_cancel_req); in btrfs_scrub_cancel()
4000 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel()
4007 struct btrfs_fs_info *fs_info = dev->fs_info; in btrfs_scrub_cancel_dev()
4010 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_cancel_dev()
4011 sctx = dev->scrub_ctx; in btrfs_scrub_cancel_dev()
4013 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel_dev()
4014 return -ENOTCONN; in btrfs_scrub_cancel_dev()
4016 atomic_inc(&sctx->cancel_req); in btrfs_scrub_cancel_dev()
4017 while (dev->scrub_ctx) { in btrfs_scrub_cancel_dev()
4018 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel_dev()
4019 wait_event(fs_info->scrub_pause_wait, in btrfs_scrub_cancel_dev()
4020 dev->scrub_ctx == NULL); in btrfs_scrub_cancel_dev()
4021 mutex_lock(&fs_info->scrub_lock); in btrfs_scrub_cancel_dev()
4023 mutex_unlock(&fs_info->scrub_lock); in btrfs_scrub_cancel_dev()
4034 mutex_lock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_progress()
4035 dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true); in btrfs_scrub_progress()
4037 sctx = dev->scrub_ctx; in btrfs_scrub_progress()
4039 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_progress()
4040 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_scrub_progress()
4042 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; in btrfs_scrub_progress()
4059 !bbio->stripes[0].dev->bdev) { in scrub_remap_extent()
4064 *extent_physical = bbio->stripes[0].physical; in scrub_remap_extent()
4065 *extent_mirror_num = bbio->mirror_num; in scrub_remap_extent()
4066 *extent_dev = bbio->stripes[0].dev; in scrub_remap_extent()