Lines Matching full:bitmap
3 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
34 #include "md-bitmap.h"
38 /* version 4 insists the bitmap is in little-endian order
47 * in-memory bitmap:
125 * bitmap structures:
128 /* the in-memory bitmap is represented by bitmap_pages */
150 /* the main bitmap structure - one per mddev */
151 struct bitmap { struct
156 /* total number of pages in the bitmap */
166 struct mddev *mddev; /* the md device that the bitmap is for */ argument
174 /* cached copy of the bitmap file superblock */ argument
183 /* total bytes in the bitmap */
196 * the bitmap daemon - periodically wakes up and sweeps the bitmap argument
201 * when we lasted called end_sync to update bitmap with resync argument
206 /* pending writes to the bitmap file */ argument
217 static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, argument
220 static inline char *bmname(struct bitmap *bitmap) in bmname() argument
222 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; in bmname()
225 static bool __bitmap_enabled(struct bitmap *bitmap) in __bitmap_enabled() argument
227 return bitmap->storage.filemap && in __bitmap_enabled()
228 !test_bit(BITMAP_STALE, &bitmap->flags); in __bitmap_enabled()
233 struct bitmap *bitmap = mddev->bitmap; in bitmap_enabled() local
235 if (!bitmap) in bitmap_enabled()
238 return __bitmap_enabled(bitmap); in bitmap_enabled()
251 static int md_bitmap_checkpage(struct bitmap_counts *bitmap, in md_bitmap_checkpage() argument
253 __releases(bitmap->lock) in md_bitmap_checkpage()
254 __acquires(bitmap->lock) in md_bitmap_checkpage()
258 WARN_ON_ONCE(page >= bitmap->pages); in md_bitmap_checkpage()
259 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ in md_bitmap_checkpage()
262 if (bitmap->bp[page].map) /* page is already allocated, just return */ in md_bitmap_checkpage()
270 spin_unlock_irq(&bitmap->lock); in md_bitmap_checkpage()
285 spin_lock_irq(&bitmap->lock); in md_bitmap_checkpage()
288 pr_debug("md/bitmap: map page allocation failed, hijacking\n"); in md_bitmap_checkpage()
294 if (!bitmap->bp[page].map) in md_bitmap_checkpage()
295 bitmap->bp[page].hijacked = 1; in md_bitmap_checkpage()
296 } else if (bitmap->bp[page].map || in md_bitmap_checkpage()
297 bitmap->bp[page].hijacked) { in md_bitmap_checkpage()
304 bitmap->bp[page].map = mappage; in md_bitmap_checkpage()
305 bitmap->missing_pages--; in md_bitmap_checkpage()
313 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page) in md_bitmap_checkfree() argument
317 if (bitmap->bp[page].count) /* page is still busy */ in md_bitmap_checkfree()
322 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ in md_bitmap_checkfree()
323 bitmap->bp[page].hijacked = 0; in md_bitmap_checkfree()
324 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
327 ptr = bitmap->bp[page].map; in md_bitmap_checkfree()
328 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
329 bitmap->missing_pages++; in md_bitmap_checkfree()
335 * bitmap file handling - read and write the bitmap file and its superblock
342 /* IO operations when bitmap is stored near all superblocks */
425 static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap, in __write_sb_page() argument
429 struct mddev *mddev = bitmap->mddev; in __write_sb_page()
430 struct bitmap_storage *store = &bitmap->storage; in __write_sb_page()
431 unsigned long num_pages = bitmap->storage.file_pages; in __write_sb_page()
455 /* Bitmap could be anywhere. */ in __write_sb_page()
460 /* DATA BITMAP METADATA */ in __write_sb_page()
463 /* bitmap runs in to metadata */ in __write_sb_page()
467 /* data runs in to bitmap */ in __write_sb_page()
470 /* METADATA BITMAP DATA */ in __write_sb_page()
473 /* bitmap runs in to data */ in __write_sb_page()
481 static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index, in write_sb_page() argument
484 struct mddev *mddev = bitmap->mddev; in write_sb_page()
490 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) { in write_sb_page()
491 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in write_sb_page()
498 static void md_bitmap_file_kick(struct bitmap *bitmap);
501 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
506 atomic_inc(&bitmap->pending_writes); in write_file_page()
514 wait_event(bitmap->write_wait, in write_file_page()
515 atomic_read(&bitmap->pending_writes) == 0); in write_file_page()
520 struct bitmap *bitmap = bh->b_private; in end_bitmap_write() local
523 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in end_bitmap_write()
524 if (atomic_dec_and_test(&bitmap->pending_writes)) in end_bitmap_write()
525 wake_up(&bitmap->write_wait); in end_bitmap_write()
553 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
561 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE, in read_file_page()
592 bh->b_private = bitmap; in read_file_page()
593 atomic_inc(&bitmap->pending_writes); in read_file_page()
602 wait_event(bitmap->write_wait, in read_file_page()
603 atomic_read(&bitmap->pending_writes)==0); in read_file_page()
604 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in read_file_page()
608 pr_err("md: bitmap read error: (%dB @ %llu): %d\n", in read_file_page()
615 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
619 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
630 * bitmap file superblock operations
636 static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index, in filemap_write_page() argument
639 struct bitmap_storage *store = &bitmap->storage; in filemap_write_page()
642 if (mddev_is_clustered(bitmap->mddev)) { in filemap_write_page()
643 /* go to node bitmap area starting point */ in filemap_write_page()
648 write_file_page(bitmap, page, wait); in filemap_write_page()
650 write_sb_page(bitmap, pg_index, page, wait); in filemap_write_page()
654 * md_bitmap_wait_writes() should be called before writing any bitmap
658 static void md_bitmap_wait_writes(struct bitmap *bitmap) in md_bitmap_wait_writes() argument
660 if (bitmap->storage.file) in md_bitmap_wait_writes()
661 wait_event(bitmap->write_wait, in md_bitmap_wait_writes()
662 atomic_read(&bitmap->pending_writes)==0); in md_bitmap_wait_writes()
667 * which is safe. The relevant bitmap blocks will in md_bitmap_wait_writes()
671 md_super_wait(bitmap->mddev); in md_bitmap_wait_writes()
679 struct bitmap *bitmap = data; in bitmap_update_sb() local
681 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */ in bitmap_update_sb()
683 if (bitmap->mddev->bitmap_info.external) in bitmap_update_sb()
685 if (!bitmap->storage.sb_page) /* no superblock */ in bitmap_update_sb()
687 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_update_sb()
688 sb->events = cpu_to_le64(bitmap->mddev->events); in bitmap_update_sb()
689 if (bitmap->mddev->events < bitmap->events_cleared) in bitmap_update_sb()
691 bitmap->events_cleared = bitmap->mddev->events; in bitmap_update_sb()
692 sb->events_cleared = cpu_to_le64(bitmap->events_cleared); in bitmap_update_sb()
695 * a bitmap write error occurred but the later writes succeeded. in bitmap_update_sb()
697 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR)); in bitmap_update_sb()
699 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); in bitmap_update_sb()
700 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); in bitmap_update_sb()
702 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in bitmap_update_sb()
703 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize); in bitmap_update_sb()
704 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes); in bitmap_update_sb()
705 sb->sectors_reserved = cpu_to_le32(bitmap->mddev-> in bitmap_update_sb()
709 if (bitmap->storage.file) in bitmap_update_sb()
710 write_file_page(bitmap, bitmap->storage.sb_page, 1); in bitmap_update_sb()
712 write_sb_page(bitmap, bitmap->storage.sb_index, in bitmap_update_sb()
713 bitmap->storage.sb_page, 1); in bitmap_update_sb()
716 static void bitmap_print_sb(struct bitmap *bitmap) in bitmap_print_sb() argument
720 if (!bitmap || !bitmap->storage.sb_page) in bitmap_print_sb()
722 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_print_sb()
723 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap)); in bitmap_print_sb()
746 * @bitmap
749 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
750 * This function verifies 'bitmap_info' and populates the on-disk bitmap
755 static int md_bitmap_new_disk_sb(struct bitmap *bitmap) in md_bitmap_new_disk_sb() argument
760 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO); in md_bitmap_new_disk_sb()
761 if (bitmap->storage.sb_page == NULL) in md_bitmap_new_disk_sb()
763 bitmap->storage.sb_index = 0; in md_bitmap_new_disk_sb()
765 sb = kmap_local_page(bitmap->storage.sb_page); in md_bitmap_new_disk_sb()
770 chunksize = bitmap->mddev->bitmap_info.chunksize; in md_bitmap_new_disk_sb()
774 pr_warn("bitmap chunksize not a power of 2\n"); in md_bitmap_new_disk_sb()
779 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep; in md_bitmap_new_disk_sb()
785 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_new_disk_sb()
791 write_behind = bitmap->mddev->bitmap_info.max_write_behind; in md_bitmap_new_disk_sb()
795 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_new_disk_sb()
797 /* keep the array size field of the bitmap superblock up to date */ in md_bitmap_new_disk_sb()
798 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_new_disk_sb()
800 memcpy(sb->uuid, bitmap->mddev->uuid, 16); in md_bitmap_new_disk_sb()
802 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_new_disk_sb()
803 sb->state = cpu_to_le32(bitmap->flags); in md_bitmap_new_disk_sb()
804 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_new_disk_sb()
805 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); in md_bitmap_new_disk_sb()
806 bitmap->mddev->bitmap_info.nodes = 0; in md_bitmap_new_disk_sb()
813 /* read the superblock from the bitmap file and initialize some bitmap fields */
814 static int md_bitmap_read_sb(struct bitmap *bitmap) in md_bitmap_read_sb() argument
826 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) { in md_bitmap_read_sb()
830 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
838 bitmap->storage.sb_page = sb_page; in md_bitmap_read_sb()
842 if (bitmap->cluster_slot >= 0) { in md_bitmap_read_sb()
843 sector_t bm_blocks = bitmap->mddev->resync_max_sectors; in md_bitmap_read_sb()
846 (bitmap->mddev->bitmap_info.chunksize >> 9)); in md_bitmap_read_sb()
851 offset = bitmap->cluster_slot * (bm_blocks << 3); in md_bitmap_read_sb()
853 bitmap->cluster_slot, offset); in md_bitmap_read_sb()
856 if (bitmap->storage.file) { in md_bitmap_read_sb()
857 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host); in md_bitmap_read_sb()
860 err = read_file_page(bitmap->storage.file, 0, in md_bitmap_read_sb()
861 bitmap, bytes, sb_page); in md_bitmap_read_sb()
863 err = read_sb_page(bitmap->mddev, offset, sb_page, 0, in md_bitmap_read_sb()
877 /* verify that the bitmap-specific fields are valid */ in md_bitmap_read_sb()
884 reason = "bitmap chunksize too small"; in md_bitmap_read_sb()
886 reason = "bitmap chunksize not a power of 2"; in md_bitmap_read_sb()
892 pr_warn("%s: invalid bitmap file superblock: %s\n", in md_bitmap_read_sb()
893 bmname(bitmap), reason); in md_bitmap_read_sb()
898 * Setup nodes/clustername only if bitmap version is in md_bitmap_read_sb()
903 strscpy(bitmap->mddev->bitmap_info.cluster_name, in md_bitmap_read_sb()
907 /* keep the array size field of the bitmap superblock up to date */ in md_bitmap_read_sb()
908 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_read_sb()
910 if (bitmap->mddev->persistent) { in md_bitmap_read_sb()
913 * bitmap's UUID and event counter to the mddev's in md_bitmap_read_sb()
915 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { in md_bitmap_read_sb()
916 pr_warn("%s: bitmap superblock UUID mismatch\n", in md_bitmap_read_sb()
917 bmname(bitmap)); in md_bitmap_read_sb()
921 if (!nodes && (events < bitmap->mddev->events)) { in md_bitmap_read_sb()
922 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n", in md_bitmap_read_sb()
923 bmname(bitmap), events, in md_bitmap_read_sb()
924 (unsigned long long) bitmap->mddev->events); in md_bitmap_read_sb()
925 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
930 bitmap->flags |= le32_to_cpu(sb->state); in md_bitmap_read_sb()
932 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags); in md_bitmap_read_sb()
933 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); in md_bitmap_read_sb()
938 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) { in md_bitmap_read_sb()
940 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
941 err = md_setup_cluster(bitmap->mddev, nodes); in md_bitmap_read_sb()
944 bmname(bitmap), err); in md_bitmap_read_sb()
947 bitmap->cluster_slot = bitmap->mddev->cluster_ops->slot_number(bitmap->mddev); in md_bitmap_read_sb()
953 if (test_bit(BITMAP_STALE, &bitmap->flags)) in md_bitmap_read_sb()
954 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_read_sb()
955 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
956 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_read_sb()
957 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_read_sb()
958 bitmap->mddev->bitmap_info.nodes = nodes; in md_bitmap_read_sb()
959 if (bitmap->mddev->bitmap_info.space == 0 || in md_bitmap_read_sb()
960 bitmap->mddev->bitmap_info.space > sectors_reserved) in md_bitmap_read_sb()
961 bitmap->mddev->bitmap_info.space = sectors_reserved; in md_bitmap_read_sb()
963 bitmap_print_sb(bitmap); in md_bitmap_read_sb()
964 if (bitmap->cluster_slot < 0) in md_bitmap_read_sb()
965 md_cluster_stop(bitmap->mddev); in md_bitmap_read_sb()
971 * general bitmap file operations
975 * on-disk bitmap:
977 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
1089 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
1093 static void md_bitmap_file_kick(struct bitmap *bitmap) in md_bitmap_file_kick() argument
1095 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) { in md_bitmap_file_kick()
1096 bitmap_update_sb(bitmap); in md_bitmap_file_kick()
1098 if (bitmap->storage.file) { in md_bitmap_file_kick()
1099 pr_warn("%s: kicking failed bitmap file %pD4 from array!\n", in md_bitmap_file_kick()
1100 bmname(bitmap), bitmap->storage.file); in md_bitmap_file_kick()
1103 pr_warn("%s: disabling internal bitmap due to errors\n", in md_bitmap_file_kick()
1104 bmname(bitmap)); in md_bitmap_file_kick()
1115 static inline void set_page_attr(struct bitmap *bitmap, int pnum, in set_page_attr() argument
1118 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in set_page_attr()
1121 static inline void clear_page_attr(struct bitmap *bitmap, int pnum, in clear_page_attr() argument
1124 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in clear_page_attr()
1127 static inline int test_page_attr(struct bitmap *bitmap, int pnum, in test_page_attr() argument
1130 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in test_page_attr()
1133 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum, in test_and_clear_page_attr() argument
1137 bitmap->storage.filemap_attr); in test_and_clear_page_attr()
1141 * to set (and eventually sync) a particular bit in the bitmap file
1146 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_set_bit() argument
1151 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_set_bit()
1152 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_set_bit()
1157 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_set_bit()
1158 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_set_bit()
1160 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
1163 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
1167 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_set_bit()
1174 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY); in md_bitmap_file_set_bit()
1177 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_clear_bit() argument
1182 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_clear_bit()
1183 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_clear_bit()
1188 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_clear_bit()
1189 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_clear_bit()
1191 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
1194 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
1196 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_clear_bit()
1201 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) { in md_bitmap_file_clear_bit()
1202 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING); in md_bitmap_file_clear_bit()
1203 bitmap->allclean = 0; in md_bitmap_file_clear_bit()
1207 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_test_bit() argument
1212 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_test_bit()
1215 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1218 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1220 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_test_bit()
1230 * sync the dirty pages of the bitmap file to disk */
1231 static void __bitmap_unplug(struct bitmap *bitmap) in __bitmap_unplug() argument
1237 if (!__bitmap_enabled(bitmap)) in __bitmap_unplug()
1242 for (i = 0; i < bitmap->storage.file_pages; i++) { in __bitmap_unplug()
1243 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in __bitmap_unplug()
1244 need_write = test_and_clear_page_attr(bitmap, i, in __bitmap_unplug()
1248 md_bitmap_wait_writes(bitmap); in __bitmap_unplug()
1249 mddev_add_trace_msg(bitmap->mddev, in __bitmap_unplug()
1252 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING); in __bitmap_unplug()
1253 filemap_write_page(bitmap, i, false); in __bitmap_unplug()
1258 md_bitmap_wait_writes(bitmap); in __bitmap_unplug()
1260 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in __bitmap_unplug()
1261 md_bitmap_file_kick(bitmap); in __bitmap_unplug()
1266 struct bitmap *bitmap; member
1275 __bitmap_unplug(unplug_work->bitmap); in md_bitmap_unplug_fn()
1279 static void bitmap_unplug_async(struct bitmap *bitmap) in bitmap_unplug_async() argument
1285 unplug_work.bitmap = bitmap; in bitmap_unplug_async()
1295 struct bitmap *bitmap = mddev->bitmap; in bitmap_unplug() local
1297 if (!bitmap) in bitmap_unplug()
1301 __bitmap_unplug(bitmap); in bitmap_unplug()
1303 bitmap_unplug_async(bitmap); in bitmap_unplug()
1306 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1309 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1310 * mapping of the bitmap file.
1312 * Special case: If there's no bitmap file, or if the bitmap file had been
1317 * This is used when reading an out-of-date bitmap.
1319 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) in md_bitmap_init_from_disk() argument
1321 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_init_from_disk()
1322 struct mddev *mddev = bitmap->mddev; in md_bitmap_init_from_disk()
1323 unsigned long chunks = bitmap->counts.chunks; in md_bitmap_init_from_disk()
1324 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_init_from_disk()
1332 /* No permanent bitmap - fill with '1s'. */ in md_bitmap_init_from_disk()
1337 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift) in md_bitmap_init_from_disk()
1339 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1340 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1347 pr_warn("%s: bitmap file too short %lu < %lu\n", in md_bitmap_init_from_disk()
1348 bmname(bitmap), in md_bitmap_init_from_disk()
1356 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE)); in md_bitmap_init_from_disk()
1369 ret = read_file_page(file, i, bitmap, count, page); in md_bitmap_init_from_disk()
1378 pr_warn("%s: bitmap file is out of date, doing full recovery\n", in md_bitmap_init_from_disk()
1379 bmname(bitmap)); in md_bitmap_init_from_disk()
1390 * If the bitmap is out of date, dirty the whole page in md_bitmap_init_from_disk()
1397 filemap_write_page(bitmap, i, true); in md_bitmap_init_from_disk()
1398 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) { in md_bitmap_init_from_disk()
1406 struct page *page = filemap_get_page(&bitmap->storage, i); in md_bitmap_init_from_disk()
1407 unsigned long bit = file_page_offset(&bitmap->storage, i); in md_bitmap_init_from_disk()
1412 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_init_from_disk()
1420 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift in md_bitmap_init_from_disk()
1422 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1423 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1429 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n", in md_bitmap_init_from_disk()
1430 bmname(bitmap), store->file_pages, in md_bitmap_init_from_disk()
1436 pr_warn("%s: bitmap initialisation failed: %d\n", in md_bitmap_init_from_disk()
1437 bmname(bitmap), ret); in md_bitmap_init_from_disk()
1441 /* just flag bitmap pages as needing to be written. */
1445 struct bitmap *bitmap = mddev->bitmap; in bitmap_write_all() local
1447 if (!bitmap || !bitmap->storage.filemap) in bitmap_write_all()
1451 if (bitmap->storage.file) in bitmap_write_all()
1454 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_write_all()
1455 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in bitmap_write_all()
1456 bitmap->allclean = 0; in bitmap_write_all()
1459 static void md_bitmap_count_page(struct bitmap_counts *bitmap, in md_bitmap_count_page() argument
1462 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_count_page()
1464 bitmap->bp[page].count += inc; in md_bitmap_count_page()
1465 md_bitmap_checkfree(bitmap, page); in md_bitmap_count_page()
1468 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset) in md_bitmap_set_pending() argument
1470 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_set_pending()
1472 struct bitmap_page *bp = &bitmap->bp[page]; in md_bitmap_set_pending()
1478 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1501 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1506 struct bitmap *bitmap; in bitmap_daemon_work() local
1516 bitmap = mddev->bitmap; in bitmap_daemon_work()
1517 if (bitmap == NULL) { in bitmap_daemon_work()
1521 if (time_before(jiffies, bitmap->daemon_lastrun in bitmap_daemon_work()
1525 bitmap->daemon_lastrun = jiffies; in bitmap_daemon_work()
1526 if (bitmap->allclean) { in bitmap_daemon_work()
1530 bitmap->allclean = 1; in bitmap_daemon_work()
1532 mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work"); in bitmap_daemon_work()
1538 for (j = 0; j < bitmap->storage.file_pages; j++) in bitmap_daemon_work()
1539 if (test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1541 set_page_attr(bitmap, j, in bitmap_daemon_work()
1544 if (bitmap->need_sync && in bitmap_daemon_work()
1549 bitmap->need_sync = 0; in bitmap_daemon_work()
1550 if (bitmap->storage.filemap) { in bitmap_daemon_work()
1551 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_daemon_work()
1553 cpu_to_le64(bitmap->events_cleared); in bitmap_daemon_work()
1555 set_page_attr(bitmap, 0, in bitmap_daemon_work()
1559 /* Now look at the bitmap counters and if any are '2' or '1', in bitmap_daemon_work()
1562 counts = &bitmap->counts; in bitmap_daemon_work()
1583 if (*bmc == 1 && !bitmap->need_sync) { in bitmap_daemon_work()
1587 md_bitmap_file_clear_bit(bitmap, block); in bitmap_daemon_work()
1591 bitmap->allclean = 0; in bitmap_daemon_work()
1596 md_bitmap_wait_writes(bitmap); in bitmap_daemon_work()
1606 j < bitmap->storage.file_pages in bitmap_daemon_work()
1607 && !test_bit(BITMAP_STALE, &bitmap->flags); in bitmap_daemon_work()
1609 if (test_page_attr(bitmap, j, in bitmap_daemon_work()
1613 if (bitmap->storage.filemap && in bitmap_daemon_work()
1614 test_and_clear_page_attr(bitmap, j, in bitmap_daemon_work()
1616 filemap_write_page(bitmap, j, false); in bitmap_daemon_work()
1620 if (bitmap->allclean == 0) in bitmap_daemon_work()
1625 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap, in md_bitmap_get_counter() argument
1628 __releases(bitmap->lock) in md_bitmap_get_counter()
1629 __acquires(bitmap->lock) in md_bitmap_get_counter()
1635 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_get_counter()
1638 sector_t csize = ((sector_t)1) << bitmap->chunkshift; in md_bitmap_get_counter()
1641 if (page >= bitmap->pages) { in md_bitmap_get_counter()
1650 err = md_bitmap_checkpage(bitmap, page, create, 0); in md_bitmap_get_counter()
1652 if (bitmap->bp[page].hijacked || in md_bitmap_get_counter()
1653 bitmap->bp[page].map == NULL) in md_bitmap_get_counter()
1654 csize = ((sector_t)1) << (bitmap->chunkshift + in md_bitmap_get_counter()
1664 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ in md_bitmap_get_counter()
1669 &bitmap->bp[page].map)[hi]; in md_bitmap_get_counter()
1672 &(bitmap->bp[page].map[pageoff]); in md_bitmap_get_counter()
1678 struct bitmap *bitmap = mddev->bitmap; in bitmap_startwrite() local
1680 if (!bitmap) in bitmap_startwrite()
1687 spin_lock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1688 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1); in bitmap_startwrite()
1690 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1700 prepare_to_wait(&bitmap->overflow_wait, &__wait, in bitmap_startwrite()
1702 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1704 finish_wait(&bitmap->overflow_wait, &__wait); in bitmap_startwrite()
1710 md_bitmap_file_set_bit(bitmap, offset); in bitmap_startwrite()
1711 md_bitmap_count_page(&bitmap->counts, offset, 1); in bitmap_startwrite()
1719 spin_unlock_irq(&bitmap->counts.lock); in bitmap_startwrite()
1733 struct bitmap *bitmap = mddev->bitmap; in bitmap_endwrite() local
1735 if (!bitmap) in bitmap_endwrite()
1743 spin_lock_irqsave(&bitmap->counts.lock, flags); in bitmap_endwrite()
1744 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0); in bitmap_endwrite()
1746 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1750 if (!bitmap->mddev->degraded) { in bitmap_endwrite()
1751 if (bitmap->events_cleared < bitmap->mddev->events) { in bitmap_endwrite()
1752 bitmap->events_cleared = bitmap->mddev->events; in bitmap_endwrite()
1753 bitmap->need_sync = 1; in bitmap_endwrite()
1755 bitmap->sysfs_can_clear); in bitmap_endwrite()
1762 wake_up(&bitmap->overflow_wait); in bitmap_endwrite()
1766 md_bitmap_set_pending(&bitmap->counts, offset); in bitmap_endwrite()
1767 bitmap->allclean = 0; in bitmap_endwrite()
1769 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in bitmap_endwrite()
1778 static bool __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, in __bitmap_start_sync() argument
1784 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */ in __bitmap_start_sync()
1786 return true; /* always resync if no bitmap */ in __bitmap_start_sync()
1788 spin_lock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1791 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_start_sync()
1804 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1824 rv |= __bitmap_start_sync(mddev->bitmap, offset, in bitmap_start_sync()
1833 static void __bitmap_end_sync(struct bitmap *bitmap, sector_t offset, in __bitmap_end_sync() argument
1839 if (bitmap == NULL) { in __bitmap_end_sync()
1843 spin_lock_irqsave(&bitmap->counts.lock, flags); in __bitmap_end_sync()
1844 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_end_sync()
1855 md_bitmap_set_pending(&bitmap->counts, offset); in __bitmap_end_sync()
1856 bitmap->allclean = 0; in __bitmap_end_sync()
1861 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in __bitmap_end_sync()
1867 __bitmap_end_sync(mddev->bitmap, offset, blocks, true); in bitmap_end_sync()
1872 /* Sync has finished, and any bitmap chunks that weren't synced in bitmap_close_sync()
1878 struct bitmap *bitmap = mddev->bitmap; in bitmap_close_sync() local
1880 if (!bitmap) in bitmap_close_sync()
1883 while (sector < bitmap->mddev->resync_max_sectors) { in bitmap_close_sync()
1884 __bitmap_end_sync(bitmap, sector, &blocks, false); in bitmap_close_sync()
1894 struct bitmap *bitmap = mddev->bitmap; in bitmap_cond_end_sync() local
1896 if (!bitmap) in bitmap_cond_end_sync()
1899 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1902 if (!force && time_before(jiffies, (bitmap->last_end_sync in bitmap_cond_end_sync()
1903 + bitmap->mddev->bitmap_info.daemon_sleep))) in bitmap_cond_end_sync()
1905 wait_event(bitmap->mddev->recovery_wait, in bitmap_cond_end_sync()
1906 atomic_read(&bitmap->mddev->recovery_active) == 0); in bitmap_cond_end_sync()
1908 bitmap->mddev->curr_resync_completed = sector; in bitmap_cond_end_sync()
1909 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags); in bitmap_cond_end_sync()
1910 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1); in bitmap_cond_end_sync()
1912 while (s < sector && s < bitmap->mddev->resync_max_sectors) { in bitmap_cond_end_sync()
1913 __bitmap_end_sync(bitmap, s, &blocks, false); in bitmap_cond_end_sync()
1916 bitmap->last_end_sync = jiffies; in bitmap_cond_end_sync()
1917 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed); in bitmap_cond_end_sync()
1924 struct bitmap *bitmap = mddev->bitmap; in bitmap_sync_with_cluster() local
1928 __bitmap_end_sync(bitmap, sector, &blocks, false); in bitmap_sync_with_cluster()
1940 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) in md_bitmap_set_memory_bits() argument
1949 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1950 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1); in md_bitmap_set_memory_bits()
1952 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1957 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_set_memory_bits()
1958 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_set_memory_bits()
1959 bitmap->allclean = 0; in md_bitmap_set_memory_bits()
1963 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1966 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1971 struct bitmap *bitmap = mddev->bitmap; in bitmap_dirty_bits() local
1973 if (!bitmap) in bitmap_dirty_bits()
1977 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift; in bitmap_dirty_bits()
1979 md_bitmap_set_memory_bits(bitmap, sec, 1); in bitmap_dirty_bits()
1980 md_bitmap_file_set_bit(bitmap, sec); in bitmap_dirty_bits()
1981 if (sec < bitmap->mddev->recovery_cp) in bitmap_dirty_bits()
1986 bitmap->mddev->recovery_cp = sec; in bitmap_dirty_bits()
1992 struct bitmap *bitmap = mddev->bitmap; in bitmap_flush() local
1995 if (!bitmap) /* there was no bitmap */ in bitmap_flush()
2002 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2004 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2006 bitmap->daemon_lastrun -= sleep; in bitmap_flush()
2010 bitmap_update_sb(bitmap); in bitmap_flush()
2017 struct bitmap *bitmap = data; in md_bitmap_free() local
2019 if (!bitmap) /* there was no bitmap */ in md_bitmap_free()
2022 if (bitmap->sysfs_can_clear) in md_bitmap_free()
2023 sysfs_put(bitmap->sysfs_can_clear); in md_bitmap_free()
2025 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info && in md_bitmap_free()
2026 bitmap->cluster_slot == bitmap->mddev->cluster_ops->slot_number(bitmap->mddev)) in md_bitmap_free()
2027 md_cluster_stop(bitmap->mddev); in md_bitmap_free()
2030 wait_event(bitmap->write_wait, in md_bitmap_free()
2031 atomic_read(&bitmap->pending_writes) == 0); in md_bitmap_free()
2033 /* release the bitmap file */ in md_bitmap_free()
2034 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_free()
2036 bp = bitmap->counts.bp; in md_bitmap_free()
2037 pages = bitmap->counts.pages; in md_bitmap_free()
2046 kfree(bitmap); in md_bitmap_free()
2051 struct bitmap *bitmap = mddev->bitmap; in bitmap_start_behind_write() local
2054 if (!bitmap) in bitmap_start_behind_write()
2057 atomic_inc(&bitmap->behind_writes); in bitmap_start_behind_write()
2058 bw = atomic_read(&bitmap->behind_writes); in bitmap_start_behind_write()
2059 if (bw > bitmap->behind_writes_used) in bitmap_start_behind_write()
2060 bitmap->behind_writes_used = bw; in bitmap_start_behind_write()
2063 bw, bitmap->mddev->bitmap_info.max_write_behind); in bitmap_start_behind_write()
2068 struct bitmap *bitmap = mddev->bitmap; in bitmap_end_behind_write() local
2070 if (!bitmap) in bitmap_end_behind_write()
2073 if (atomic_dec_and_test(&bitmap->behind_writes)) in bitmap_end_behind_write()
2074 wake_up(&bitmap->behind_wait); in bitmap_end_behind_write()
2076 atomic_read(&bitmap->behind_writes), in bitmap_end_behind_write()
2077 bitmap->mddev->bitmap_info.max_write_behind); in bitmap_end_behind_write()
2082 struct bitmap *bitmap = mddev->bitmap; in bitmap_wait_behind_writes() local
2085 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { in bitmap_wait_behind_writes()
2089 wait_event(bitmap->behind_wait, in bitmap_wait_behind_writes()
2090 atomic_read(&bitmap->behind_writes) == 0); in bitmap_wait_behind_writes()
2096 struct bitmap *bitmap = mddev->bitmap; in bitmap_destroy() local
2098 if (!bitmap) /* there was no bitmap */ in bitmap_destroy()
2107 mddev->bitmap = NULL; /* disconnect from the md device */ in bitmap_destroy()
2112 md_bitmap_free(bitmap); in bitmap_destroy()
2116 * initialize the bitmap structure
2118 * once mddev->bitmap is set
2120 static struct bitmap *__bitmap_create(struct mddev *mddev, int slot) in __bitmap_create()
2122 struct bitmap *bitmap; in __bitmap_create() local
2133 pr_notice("md/raid:%s: array with journal cannot have bitmap\n", in __bitmap_create()
2138 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); in __bitmap_create()
2139 if (!bitmap) in __bitmap_create()
2142 spin_lock_init(&bitmap->counts.lock); in __bitmap_create()
2143 atomic_set(&bitmap->pending_writes, 0); in __bitmap_create()
2144 init_waitqueue_head(&bitmap->write_wait); in __bitmap_create()
2145 init_waitqueue_head(&bitmap->overflow_wait); in __bitmap_create()
2146 init_waitqueue_head(&bitmap->behind_wait); in __bitmap_create()
2148 bitmap->mddev = mddev; in __bitmap_create()
2149 bitmap->cluster_slot = slot; in __bitmap_create()
2152 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap"); in __bitmap_create()
2154 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); in __bitmap_create()
2157 bitmap->sysfs_can_clear = NULL; in __bitmap_create()
2159 bitmap->storage.file = file; in __bitmap_create()
2168 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */ in __bitmap_create()
2172 * instructing us to create a new on-disk bitmap instance. in __bitmap_create()
2175 err = md_bitmap_new_disk_sb(bitmap); in __bitmap_create()
2177 err = md_bitmap_read_sb(bitmap); in __bitmap_create()
2189 bitmap->daemon_lastrun = jiffies; in __bitmap_create()
2190 err = __bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, in __bitmap_create()
2195 pr_debug("created bitmap (%lu pages) for device %s\n", in __bitmap_create()
2196 bitmap->counts.pages, bmname(bitmap)); in __bitmap_create()
2198 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0; in __bitmap_create()
2202 return bitmap; in __bitmap_create()
2204 md_bitmap_free(bitmap); in __bitmap_create()
2210 struct bitmap *bitmap = __bitmap_create(mddev, slot); in bitmap_create() local
2212 if (IS_ERR(bitmap)) in bitmap_create()
2213 return PTR_ERR(bitmap); in bitmap_create()
2215 mddev->bitmap = bitmap; in bitmap_create()
2224 struct bitmap *bitmap = mddev->bitmap; in bitmap_load() local
2227 if (!bitmap) in bitmap_load()
2236 /* Clear out old bitmap info first: Either there is none, or we in bitmap_load()
2249 || bitmap->events_cleared == mddev->events) in bitmap_load()
2255 err = md_bitmap_init_from_disk(bitmap, start); in bitmap_load()
2260 clear_bit(BITMAP_STALE, &bitmap->flags); in bitmap_load()
2263 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); in bitmap_load()
2268 bitmap_update_sb(bitmap); in bitmap_load()
2270 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in bitmap_load()
2276 /* caller need to free returned bitmap with md_bitmap_free() */
2280 struct bitmap *bitmap; in bitmap_get_from_slot() local
2282 bitmap = __bitmap_create(mddev, slot); in bitmap_get_from_slot()
2283 if (IS_ERR(bitmap)) { in bitmap_get_from_slot()
2284 rv = PTR_ERR(bitmap); in bitmap_get_from_slot()
2288 rv = md_bitmap_init_from_disk(bitmap, 0); in bitmap_get_from_slot()
2290 md_bitmap_free(bitmap); in bitmap_get_from_slot()
2294 return bitmap; in bitmap_get_from_slot()
2297 /* Loads the bitmap associated with slot and copies the resync information
2298 * to our bitmap
2306 struct bitmap *bitmap; in bitmap_copy_from_slot() local
2308 bitmap = bitmap_get_from_slot(mddev, slot); in bitmap_copy_from_slot()
2309 if (IS_ERR(bitmap)) { in bitmap_copy_from_slot()
2310 pr_err("%s can't get bitmap from slot %d\n", __func__, slot); in bitmap_copy_from_slot()
2314 counts = &bitmap->counts; in bitmap_copy_from_slot()
2317 if (md_bitmap_file_test_bit(bitmap, block)) { in bitmap_copy_from_slot()
2321 md_bitmap_file_clear_bit(bitmap, block); in bitmap_copy_from_slot()
2322 md_bitmap_set_memory_bits(mddev->bitmap, block, 1); in bitmap_copy_from_slot()
2323 md_bitmap_file_set_bit(mddev->bitmap, block); in bitmap_copy_from_slot()
2328 bitmap_update_sb(bitmap); in bitmap_copy_from_slot()
2331 for (i = 0; i < bitmap->storage.file_pages; i++) in bitmap_copy_from_slot()
2332 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING)) in bitmap_copy_from_slot()
2333 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in bitmap_copy_from_slot()
2334 __bitmap_unplug(bitmap); in bitmap_copy_from_slot()
2336 __bitmap_unplug(mddev->bitmap); in bitmap_copy_from_slot()
2339 md_bitmap_free(bitmap); in bitmap_copy_from_slot()
2346 struct bitmap *bitmap = data; in bitmap_set_pages() local
2348 bitmap->counts.pages = pages; in bitmap_set_pages()
2355 struct bitmap *bitmap = data; in bitmap_get_stats() local
2358 if (!bitmap) in bitmap_get_stats()
2360 if (!bitmap->mddev->bitmap_info.external && in bitmap_get_stats()
2361 !bitmap->storage.sb_page) in bitmap_get_stats()
2363 sb = kmap_local_page(bitmap->storage.sb_page); in bitmap_get_stats()
2367 counts = &bitmap->counts; in bitmap_get_stats()
2371 storage = &bitmap->storage; in bitmap_get_stats()
2375 stats->behind_writes = atomic_read(&bitmap->behind_writes); in bitmap_get_stats()
2376 stats->behind_wait = wq_has_sleeper(&bitmap->behind_wait); in bitmap_get_stats()
2377 stats->events_cleared = bitmap->events_cleared; in bitmap_get_stats()
2381 static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, in __bitmap_resize() argument
2386 * Then quiesce, copy bits, replace bitmap, and re-start in __bitmap_resize()
2388 * This function is called both to set up the initial bitmap in __bitmap_resize()
2389 * and to resize the bitmap while the array is active. in __bitmap_resize()
2404 if (bitmap->storage.file && !init) { in __bitmap_resize()
2405 pr_info("md: cannot resize file-based bitmap\n"); in __bitmap_resize()
2414 long space = bitmap->mddev->bitmap_info.space; in __bitmap_resize()
2420 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8); in __bitmap_resize()
2421 if (!bitmap->mddev->bitmap_info.external) in __bitmap_resize()
2424 bitmap->mddev->bitmap_info.space = space; in __bitmap_resize()
2426 chunkshift = bitmap->counts.chunkshift; in __bitmap_resize()
2433 if (!bitmap->mddev->bitmap_info.external) in __bitmap_resize()
2442 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file) in __bitmap_resize()
2444 !bitmap->mddev->bitmap_info.external, in __bitmap_resize()
2445 mddev_is_clustered(bitmap->mddev) in __bitmap_resize()
2446 ? bitmap->cluster_slot : 0); in __bitmap_resize()
2462 bitmap->mddev->pers->quiesce(bitmap->mddev, 1); in __bitmap_resize()
2464 store.file = bitmap->storage.file; in __bitmap_resize()
2465 bitmap->storage.file = NULL; in __bitmap_resize()
2467 if (store.sb_page && bitmap->storage.sb_page) in __bitmap_resize()
2469 page_address(bitmap->storage.sb_page), in __bitmap_resize()
2471 spin_lock_irq(&bitmap->counts.lock); in __bitmap_resize()
2472 md_bitmap_file_unmap(&bitmap->storage); in __bitmap_resize()
2473 bitmap->storage = store; in __bitmap_resize()
2475 old_counts = bitmap->counts; in __bitmap_resize()
2476 bitmap->counts.bp = new_bp; in __bitmap_resize()
2477 bitmap->counts.pages = pages; in __bitmap_resize()
2478 bitmap->counts.missing_pages = pages; in __bitmap_resize()
2479 bitmap->counts.chunkshift = chunkshift; in __bitmap_resize()
2480 bitmap->counts.chunks = chunks; in __bitmap_resize()
2481 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift + in __bitmap_resize()
2487 /* For cluster raid, need to pre-allocate bitmap */ in __bitmap_resize()
2488 if (mddev_is_clustered(bitmap->mddev)) { in __bitmap_resize()
2491 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1); in __bitmap_resize()
2502 bitmap->counts.bp = old_counts.bp; in __bitmap_resize()
2503 bitmap->counts.pages = old_counts.pages; in __bitmap_resize()
2504 bitmap->counts.missing_pages = old_counts.pages; in __bitmap_resize()
2505 bitmap->counts.chunkshift = old_counts.chunkshift; in __bitmap_resize()
2506 bitmap->counts.chunks = old_counts.chunks; in __bitmap_resize()
2507 bitmap->mddev->bitmap_info.chunksize = in __bitmap_resize()
2510 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n"); in __bitmap_resize()
2513 bitmap->counts.bp[page].count += 1; in __bitmap_resize()
2525 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in __bitmap_resize()
2534 md_bitmap_file_set_bit(bitmap, block); in __bitmap_resize()
2538 md_bitmap_count_page(&bitmap->counts, block, 1); in __bitmap_resize()
2539 md_bitmap_set_pending(&bitmap->counts, block); in __bitmap_resize()
2549 if (bitmap->counts.bp != old_counts.bp) { in __bitmap_resize()
2561 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in __bitmap_resize()
2568 md_bitmap_count_page(&bitmap->counts, block, 1); in __bitmap_resize()
2569 md_bitmap_set_pending(&bitmap->counts, block); in __bitmap_resize()
2574 for (i = 0; i < bitmap->storage.file_pages; i++) in __bitmap_resize()
2575 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in __bitmap_resize()
2577 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_resize()
2580 __bitmap_unplug(bitmap); in __bitmap_resize()
2581 bitmap->mddev->pers->quiesce(bitmap->mddev, 0); in __bitmap_resize()
2591 struct bitmap *bitmap = mddev->bitmap; in bitmap_resize() local
2593 if (!bitmap) in bitmap_resize()
2596 return __bitmap_resize(bitmap, blocks, chunksize, init); in bitmap_resize()
2629 if (mddev->bitmap || mddev->bitmap_info.file || in location_store()
2631 /* bitmap already configured. Only option is to clear it */ in location_store()
2645 /* No bitmap, OK to set a location */ in location_store()
2686 /* Ensure new bitmap info is stored in in location_store()
2703 /* 'bitmap/space' is the space available at 'location' for the
2704 * bitmap. This allows the kernel to know when it is safe to
2705 * resize the bitmap to match a resized array.
2716 struct bitmap *bitmap; in space_store() local
2727 bitmap = mddev->bitmap; in space_store()
2728 if (bitmap && sectors < (bitmap->storage.bytes + 511) >> 9) in space_store()
2729 return -EFBIG; /* Bitmap is too big for this small space */ in space_store()
2836 bitmap_update_sb(mddev->bitmap); in backlog_store()
2854 /* Can only be changed when no bitmap is active */ in chunksize_store()
2857 if (mddev->bitmap) in chunksize_store()
2885 if (mddev->bitmap || in metadata_store()
2905 struct bitmap *bitmap; in can_clear_show() local
2908 bitmap = mddev->bitmap; in can_clear_show()
2909 if (bitmap) in can_clear_show()
2910 len = sprintf(page, "%s\n", (bitmap->need_sync ? "false" : in can_clear_show()
2920 struct bitmap *bitmap = mddev->bitmap; in can_clear_store() local
2922 if (!bitmap) in can_clear_store()
2926 bitmap->need_sync = 1; in can_clear_store()
2933 bitmap->need_sync = 0; in can_clear_store()
2947 struct bitmap *bitmap; in behind_writes_used_show() local
2950 bitmap = mddev->bitmap; in behind_writes_used_show()
2951 if (!bitmap) in behind_writes_used_show()
2954 ret = sprintf(page, "%lu\n", bitmap->behind_writes_used); in behind_writes_used_show()
2963 struct bitmap *bitmap = mddev->bitmap; in behind_writes_used_reset() local
2965 if (bitmap) in behind_writes_used_reset()
2966 bitmap->behind_writes_used = 0; in behind_writes_used_reset()
2986 .name = "bitmap",