Lines Matching +full:- +full:- +full:commits
1 // SPDX-License-Identifier: GPL-2.0
8 * Ext4 fast commits routines.
16 * Ext4 Fast Commits
17 * -----------------
19 * Ext4 fast commits implement fine grained journalling for Ext4.
21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
24 * don't have replay code, fast commit falls back to full commits.
25 * Fast commits record delta in one of the following three categories.
29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
30 * - EXT4_FC_TAG_LINK - records directory entry link
31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
45 * ----------------
46 * With fast commits, we maintain all the directory entry operations in the
47 * order in which they are issued in an in-memory queue. This queue is flushed
67 * -------------------------
69 * Not all operations are supported by fast commits today (e.g extended
74 * Atomicity of commits
75 * --------------------
84 * - Create a new file A and remove existing file B
85 * - fsync()
86 * - Append contents to file A
87 * - Truncate file A
88 * - fsync()
92 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
97 * ------------------------------
99 * Fast commits tags are idempotent in nature provided the recovery code follows
109 * - Link dirent b to inode 10
110 * - Unlink dirent a
111 * - Inode <10> with valid refcount
117 * commits make it idempotent. Consider following sequence of operations:
128 * the procedure fast commits store the outcome of each procedure. Thus the fast
142 * similarly. Thus, by converting a non-idempotent procedure into a series of
143 * idempotent outcomes, fast commits ensured idempotence during the replay.
146 * -----
157 * commits.
175 ext4_debug("%s: Block %lld up-to-date", in ext4_end_buffer_io_sync()
176 __func__, bh->b_blocknr); in ext4_end_buffer_io_sync()
179 ext4_debug("%s: Block %lld not up-to-date", in ext4_end_buffer_io_sync()
180 __func__, bh->b_blocknr); in ext4_end_buffer_io_sync()
191 ei->i_fc_lblk_start = 0; in ext4_fc_reset_inode()
192 ei->i_fc_lblk_len = 0; in ext4_fc_reset_inode()
201 INIT_LIST_HEAD(&ei->i_fc_list); in ext4_fc_init_inode()
202 INIT_LIST_HEAD(&ei->i_fc_dilist); in ext4_fc_init_inode()
203 init_waitqueue_head(&ei->i_fc_wait); in ext4_fc_init_inode()
204 atomic_set(&ei->i_fc_updates, 0); in ext4_fc_init_inode()
207 /* This function must be called with sbi->s_fc_lock held. */
209 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock) in ext4_fc_wait_committing_inode()
215 DEFINE_WAIT_BIT(wait, &ei->i_state_flags, in ext4_fc_wait_committing_inode()
217 wq = bit_waitqueue(&ei->i_state_flags, in ext4_fc_wait_committing_inode()
220 DEFINE_WAIT_BIT(wait, &ei->i_flags, in ext4_fc_wait_committing_inode()
222 wq = bit_waitqueue(&ei->i_flags, in ext4_fc_wait_committing_inode()
225 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_wait_committing_inode()
227 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_wait_committing_inode()
235 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); in ext4_fc_disabled()
249 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_start_update()
253 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_start_update()
254 if (list_empty(&ei->i_fc_list)) in ext4_fc_start_update()
262 atomic_inc(&ei->i_fc_updates); in ext4_fc_start_update()
263 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_start_update()
267 * Stop inode update and wake up waiting fast commits if any.
273 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_stop_update()
276 if (atomic_dec_and_test(&ei->i_fc_updates)) in ext4_fc_stop_update()
277 wake_up_all(&ei->i_fc_wait); in ext4_fc_stop_update()
287 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); in ext4_fc_del()
290 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_del()
294 spin_lock(&sbi->s_fc_lock); in ext4_fc_del()
295 if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { in ext4_fc_del()
296 spin_unlock(&sbi->s_fc_lock); in ext4_fc_del()
305 if (!list_empty(&ei->i_fc_list)) in ext4_fc_del()
306 list_del_init(&ei->i_fc_list); in ext4_fc_del()
312 if (list_empty(&ei->i_fc_dilist)) { in ext4_fc_del()
313 spin_unlock(&sbi->s_fc_lock); in ext4_fc_del()
317 fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist); in ext4_fc_del()
318 WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); in ext4_fc_del()
319 list_del_init(&fc_dentry->fcd_list); in ext4_fc_del()
320 list_del_init(&fc_dentry->fcd_dilist); in ext4_fc_del()
322 WARN_ON(!list_empty(&ei->i_fc_dilist)); in ext4_fc_del()
323 spin_unlock(&sbi->s_fc_lock); in ext4_fc_del()
325 release_dentry_name_snapshot(&fc_dentry->fcd_name); in ext4_fc_del()
347 tid = handle->h_transaction->t_tid; in ext4_fc_mark_ineligible()
349 read_lock(&sbi->s_journal->j_state_lock); in ext4_fc_mark_ineligible()
350 if (sbi->s_journal->j_running_transaction) in ext4_fc_mark_ineligible()
351 tid = sbi->s_journal->j_running_transaction->t_tid; in ext4_fc_mark_ineligible()
354 read_unlock(&sbi->s_journal->j_state_lock); in ext4_fc_mark_ineligible()
356 spin_lock(&sbi->s_fc_lock); in ext4_fc_mark_ineligible()
358 if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) in ext4_fc_mark_ineligible()
359 sbi->s_fc_ineligible_tid = tid; in ext4_fc_mark_ineligible()
361 spin_unlock(&sbi->s_fc_lock); in ext4_fc_mark_ineligible()
363 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; in ext4_fc_mark_ineligible()
383 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); in ext4_fc_track_template()
387 tid = handle->h_transaction->t_tid; in ext4_fc_track_template()
388 mutex_lock(&ei->i_fc_lock); in ext4_fc_track_template()
389 if (tid == ei->i_sync_tid) { in ext4_fc_track_template()
393 ei->i_sync_tid = tid; in ext4_fc_track_template()
396 mutex_unlock(&ei->i_fc_lock); in ext4_fc_track_template()
401 spin_lock(&sbi->s_fc_lock); in ext4_fc_track_template()
402 if (list_empty(&EXT4_I(inode)->i_fc_list)) in ext4_fc_track_template()
403 list_add_tail(&EXT4_I(inode)->i_fc_list, in ext4_fc_track_template()
404 (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || in ext4_fc_track_template()
405 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? in ext4_fc_track_template()
406 &sbi->s_fc_q[FC_Q_STAGING] : in ext4_fc_track_template()
407 &sbi->s_fc_q[FC_Q_MAIN]); in ext4_fc_track_template()
408 spin_unlock(&sbi->s_fc_lock); in ext4_fc_track_template()
418 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
426 struct dentry *dentry = dentry_update->dentry; in __track_dentry_update()
427 struct inode *dir = dentry->d_parent->d_inode; in __track_dentry_update()
428 struct super_block *sb = inode->i_sb; in __track_dentry_update()
431 mutex_unlock(&ei->i_fc_lock); in __track_dentry_update()
436 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
437 return -EOPNOTSUPP; in __track_dentry_update()
443 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
444 return -ENOMEM; in __track_dentry_update()
447 node->fcd_op = dentry_update->op; in __track_dentry_update()
448 node->fcd_parent = dir->i_ino; in __track_dentry_update()
449 node->fcd_ino = inode->i_ino; in __track_dentry_update()
450 take_dentry_name_snapshot(&node->fcd_name, dentry); in __track_dentry_update()
451 INIT_LIST_HEAD(&node->fcd_dilist); in __track_dentry_update()
452 spin_lock(&sbi->s_fc_lock); in __track_dentry_update()
453 if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || in __track_dentry_update()
454 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) in __track_dentry_update()
455 list_add_tail(&node->fcd_list, in __track_dentry_update()
456 &sbi->s_fc_dentry_q[FC_Q_STAGING]); in __track_dentry_update()
458 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); in __track_dentry_update()
466 * sbi->s_fc_q to get the corresponding inode in in __track_dentry_update()
469 if (dentry_update->op == EXT4_FC_TAG_CREAT) { in __track_dentry_update()
470 WARN_ON(!list_empty(&ei->i_fc_dilist)); in __track_dentry_update()
471 list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); in __track_dentry_update()
473 spin_unlock(&sbi->s_fc_lock); in __track_dentry_update()
474 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
497 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_track_unlink()
500 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) in ext4_fc_track_unlink()
524 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_track_link()
527 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) in ext4_fc_track_link()
551 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_track_create()
554 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) in ext4_fc_track_create()
565 return -EEXIST; in __track_inode()
567 EXT4_I(inode)->i_fc_lblk_len = 0; in __track_inode()
576 if (S_ISDIR(inode->i_mode)) in ext4_fc_track_inode()
579 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_track_inode()
583 ext4_fc_mark_ineligible(inode->i_sb, in ext4_fc_track_inode()
588 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) in ext4_fc_track_inode()
608 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { in __track_range()
609 ext4_debug("Special inode %ld being modified\n", inode->i_ino); in __track_range()
610 return -ECANCELED; in __track_range()
613 oldstart = ei->i_fc_lblk_start; in __track_range()
615 if (update && ei->i_fc_lblk_len > 0) { in __track_range()
616 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); in __track_range()
617 ei->i_fc_lblk_len = in __track_range()
618 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - in __track_range()
619 ei->i_fc_lblk_start + 1; in __track_range()
621 ei->i_fc_lblk_start = __arg->start; in __track_range()
622 ei->i_fc_lblk_len = __arg->end - __arg->start + 1; in __track_range()
634 if (S_ISDIR(inode->i_mode)) in ext4_fc_track_range()
637 if (ext4_fc_disabled(inode->i_sb)) in ext4_fc_track_range()
640 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) in ext4_fc_track_range()
644 ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, in ext4_fc_track_range()
660 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; in ext4_fc_submit_bh()
668 bh->b_end_io = ext4_end_buffer_io_sync; in ext4_fc_submit_bh()
670 EXT4_SB(sb)->s_fc_bh = NULL; in ext4_fc_submit_bh()
691 int bsize = sbi->s_journal->j_blocksize; in ext4_fc_reserve_space()
692 int ret, off = sbi->s_fc_bytes % bsize; in ext4_fc_reserve_space()
700 if (len > bsize - EXT4_FC_TAG_BASE_LEN) in ext4_fc_reserve_space()
703 if (!sbi->s_fc_bh) { in ext4_fc_reserve_space()
704 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); in ext4_fc_reserve_space()
707 sbi->s_fc_bh = bh; in ext4_fc_reserve_space()
709 dst = sbi->s_fc_bh->b_data + off; in ext4_fc_reserve_space()
715 remaining = bsize - EXT4_FC_TAG_BASE_LEN - off; in ext4_fc_reserve_space()
717 sbi->s_fc_bytes += len; in ext4_fc_reserve_space()
730 *crc = ext4_chksum(sbi, *crc, sbi->s_fc_bh->b_data, bsize); in ext4_fc_reserve_space()
734 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); in ext4_fc_reserve_space()
737 sbi->s_fc_bh = bh; in ext4_fc_reserve_space()
738 sbi->s_fc_bytes += bsize - off + len; in ext4_fc_reserve_space()
739 return sbi->s_fc_bh->b_data; in ext4_fc_reserve_space()
755 int off, bsize = sbi->s_journal->j_blocksize; in ext4_fc_write_tail()
764 return -ENOSPC; in ext4_fc_write_tail()
766 off = sbi->s_fc_bytes % bsize; in ext4_fc_write_tail()
769 tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail)); in ext4_fc_write_tail()
770 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); in ext4_fc_write_tail()
774 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); in ext4_fc_write_tail()
777 crc = ext4_chksum(sbi, crc, sbi->s_fc_bh->b_data, in ext4_fc_write_tail()
778 dst - (u8 *)sbi->s_fc_bh->b_data); in ext4_fc_write_tail()
782 memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */ in ext4_fc_write_tail()
818 int dlen = fc_dentry->fcd_name.name.len; in ext4_fc_add_dentry_tlv()
825 fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent); in ext4_fc_add_dentry_tlv()
826 fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); in ext4_fc_add_dentry_tlv()
827 tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); in ext4_fc_add_dentry_tlv()
833 memcpy(dst, fc_dentry->fcd_name.name.name, dlen); in ext4_fc_add_dentry_tlv()
857 inode_len = EXT4_INODE_SIZE(inode->i_sb); in ext4_fc_write_inode()
858 else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) in ext4_fc_write_inode()
859 inode_len += ei->i_extra_isize; in ext4_fc_write_inode()
861 fc_inode.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode()
865 ret = -ECANCELED; in ext4_fc_write_inode()
866 dst = ext4_fc_reserve_space(inode->i_sb, in ext4_fc_write_inode()
896 mutex_lock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
897 if (ei->i_fc_lblk_len == 0) { in ext4_fc_write_inode_data()
898 mutex_unlock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
901 old_blk_size = ei->i_fc_lblk_start; in ext4_fc_write_inode_data()
902 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; in ext4_fc_write_inode_data()
903 ei->i_fc_lblk_len = 0; in ext4_fc_write_inode_data()
904 mutex_unlock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
908 cur_lblk_off, new_blk_size, inode->i_ino); in ext4_fc_write_inode_data()
912 map.m_len = new_blk_size - cur_lblk_off + 1; in ext4_fc_write_inode_data()
915 return -ECANCELED; in ext4_fc_write_inode_data()
923 lrange.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode_data()
926 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, in ext4_fc_write_inode_data()
928 return -ENOSPC; in ext4_fc_write_inode_data()
936 fc_ext.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode_data()
938 ex->ee_block = cpu_to_le32(map.m_lblk); in ext4_fc_write_inode_data()
939 ex->ee_len = cpu_to_le16(map.m_len); in ext4_fc_write_inode_data()
945 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, in ext4_fc_write_inode_data()
947 return -ENOSPC; in ext4_fc_write_inode_data()
960 struct super_block *sb = journal->j_private; in ext4_fc_submit_inode_data_all()
965 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
966 list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { in ext4_fc_submit_inode_data_all()
967 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); in ext4_fc_submit_inode_data_all()
968 while (atomic_read(&ei->i_fc_updates)) { in ext4_fc_submit_inode_data_all()
971 prepare_to_wait(&ei->i_fc_wait, &wait, in ext4_fc_submit_inode_data_all()
973 if (atomic_read(&ei->i_fc_updates)) { in ext4_fc_submit_inode_data_all()
974 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
976 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
978 finish_wait(&ei->i_fc_wait, &wait); in ext4_fc_submit_inode_data_all()
980 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
981 ret = jbd2_submit_inode_data(journal, ei->jinode); in ext4_fc_submit_inode_data_all()
984 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
986 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
994 struct super_block *sb = journal->j_private; in ext4_fc_wait_inode_data_all()
999 spin_lock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
1000 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { in ext4_fc_wait_inode_data_all()
1001 if (!ext4_test_inode_state(&pos->vfs_inode, in ext4_fc_wait_inode_data_all()
1004 spin_unlock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
1006 ret = jbd2_wait_inode_data(journal, pos->jinode); in ext4_fc_wait_inode_data_all()
1009 spin_lock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
1011 spin_unlock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
1018 __acquires(&sbi->s_fc_lock) in ext4_fc_commit_dentry_updates()
1019 __releases(&sbi->s_fc_lock) in ext4_fc_commit_dentry_updates()
1021 struct super_block *sb = journal->j_private; in ext4_fc_commit_dentry_updates()
1028 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) in ext4_fc_commit_dentry_updates()
1031 &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { in ext4_fc_commit_dentry_updates()
1032 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { in ext4_fc_commit_dentry_updates()
1033 spin_unlock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1035 ret = -ENOSPC; in ext4_fc_commit_dentry_updates()
1038 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1042 * With fcd_dilist we need not loop in sbi->s_fc_q to get the in ext4_fc_commit_dentry_updates()
1045 WARN_ON(list_empty(&fc_dentry->fcd_dilist)); in ext4_fc_commit_dentry_updates()
1046 ei = list_first_entry(&fc_dentry->fcd_dilist, in ext4_fc_commit_dentry_updates()
1048 inode = &ei->vfs_inode; in ext4_fc_commit_dentry_updates()
1049 WARN_ON(inode->i_ino != fc_dentry->fcd_ino); in ext4_fc_commit_dentry_updates()
1051 spin_unlock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1069 ret = -ENOSPC; in ext4_fc_commit_dentry_updates()
1073 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1077 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1083 struct super_block *sb = journal->j_private; in ext4_fc_perform_commit()
1104 if (journal->j_fs_dev != journal->j_dev) in ext4_fc_perform_commit()
1105 blkdev_issue_flush(journal->j_fs_dev); in ext4_fc_perform_commit()
1108 if (sbi->s_fc_bytes == 0) { in ext4_fc_perform_commit()
1115 sbi->s_journal->j_running_transaction->t_tid); in ext4_fc_perform_commit()
1118 ret = -ENOSPC; in ext4_fc_perform_commit()
1123 spin_lock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1126 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1130 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { in ext4_fc_perform_commit()
1131 inode = &iter->vfs_inode; in ext4_fc_perform_commit()
1135 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1142 spin_lock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1144 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1156 struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats; in ext4_fc_update_stats()
1161 stats->fc_num_commits++; in ext4_fc_update_stats()
1162 stats->fc_numblks += nblks; in ext4_fc_update_stats()
1163 if (likely(stats->s_fc_avg_commit_time)) in ext4_fc_update_stats()
1164 stats->s_fc_avg_commit_time = in ext4_fc_update_stats()
1166 stats->s_fc_avg_commit_time * 3) / 4; in ext4_fc_update_stats()
1168 stats->s_fc_avg_commit_time = commit_time; in ext4_fc_update_stats()
1172 stats->fc_failed_commits++; in ext4_fc_update_stats()
1173 stats->fc_ineligible_commits++; in ext4_fc_update_stats()
1175 stats->fc_skipped_commits++; in ext4_fc_update_stats()
1188 struct super_block *sb = journal->j_private; in ext4_fc_commit()
1190 int nblks = 0, ret, bsize = journal->j_blocksize; in ext4_fc_commit()
1191 int subtid = atomic_read(&sbi->s_fc_subtid); in ext4_fc_commit()
1204 if (ret == -EALREADY) { in ext4_fc_commit()
1206 if (atomic_read(&sbi->s_fc_subtid) <= subtid && in ext4_fc_commit()
1207 tid_gt(commit_tid, journal->j_commit_sequence)) in ext4_fc_commit()
1231 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; in ext4_fc_commit()
1237 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; in ext4_fc_commit()
1243 atomic_inc(&sbi->s_fc_subtid); in ext4_fc_commit()
1265 struct super_block *sb = journal->j_private; in ext4_fc_cleanup()
1270 if (full && sbi->s_fc_bh) in ext4_fc_cleanup()
1271 sbi->s_fc_bh = NULL; in ext4_fc_cleanup()
1276 spin_lock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1277 list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN], in ext4_fc_cleanup()
1279 list_del_init(&iter->i_fc_list); in ext4_fc_cleanup()
1280 ext4_clear_inode_state(&iter->vfs_inode, in ext4_fc_cleanup()
1282 if (tid_geq(tid, iter->i_sync_tid)) { in ext4_fc_cleanup()
1283 ext4_fc_reset_inode(&iter->vfs_inode); in ext4_fc_cleanup()
1287 * modified while the commit was running. Re-enqueue in ext4_fc_cleanup()
1294 list_add_tail(&EXT4_I(&iter->vfs_inode)->i_fc_list, in ext4_fc_cleanup()
1295 &sbi->s_fc_q[FC_Q_STAGING]); in ext4_fc_cleanup()
1300 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING); in ext4_fc_cleanup()
1302 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING); in ext4_fc_cleanup()
1306 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { in ext4_fc_cleanup()
1307 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], in ext4_fc_cleanup()
1310 list_del_init(&fc_dentry->fcd_list); in ext4_fc_cleanup()
1311 list_del_init(&fc_dentry->fcd_dilist); in ext4_fc_cleanup()
1312 spin_unlock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1314 release_dentry_name_snapshot(&fc_dentry->fcd_name); in ext4_fc_cleanup()
1316 spin_lock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1319 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], in ext4_fc_cleanup()
1320 &sbi->s_fc_dentry_q[FC_Q_MAIN]); in ext4_fc_cleanup()
1321 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], in ext4_fc_cleanup()
1322 &sbi->s_fc_q[FC_Q_MAIN]); in ext4_fc_cleanup()
1324 if (tid_geq(tid, sbi->s_fc_ineligible_tid)) { in ext4_fc_cleanup()
1325 sbi->s_fc_ineligible_tid = 0; in ext4_fc_cleanup()
1330 sbi->s_fc_bytes = 0; in ext4_fc_cleanup()
1331 spin_unlock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1356 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); in tl_to_darg()
1357 darg->ino = le32_to_cpu(fcd.fc_ino); in tl_to_darg()
1358 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); in tl_to_darg()
1359 darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); in tl_to_darg()
1367 tl->fc_len = le16_to_cpu(tl_disk.fc_len); in ext4_fc_get_tl()
1368 tl->fc_tag = le16_to_cpu(tl_disk.fc_tag); in ext4_fc_get_tl()
1403 /* -ENOENT ok coz it might not exist anymore. */ in ext4_fc_replay_unlink()
1404 if (ret == -ENOENT) in ext4_fc_replay_unlink()
1417 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); in ext4_fc_replay_link_internal()
1420 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL); in ext4_fc_replay_link_internal()
1422 ext4_debug("Dir with inode %d not found.", darg->parent_ino); in ext4_fc_replay_link_internal()
1437 ret = -ENOMEM; in ext4_fc_replay_link_internal()
1448 if (ret && ret != -EEXIST) { in ext4_fc_replay_link_internal()
1501 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_record_modified_inode()
1502 for (i = 0; i < state->fc_modified_inodes_used; i++) in ext4_fc_record_modified_inode()
1503 if (state->fc_modified_inodes[i] == ino) in ext4_fc_record_modified_inode()
1505 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { in ext4_fc_record_modified_inode()
1508 fc_modified_inodes = krealloc(state->fc_modified_inodes, in ext4_fc_record_modified_inode()
1509 sizeof(int) * (state->fc_modified_inodes_size + in ext4_fc_record_modified_inode()
1513 return -ENOMEM; in ext4_fc_record_modified_inode()
1514 state->fc_modified_inodes = fc_modified_inodes; in ext4_fc_record_modified_inode()
1515 state->fc_modified_inodes_size += in ext4_fc_record_modified_inode()
1518 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino; in ext4_fc_record_modified_inode()
1533 int inode_len, ino, ret, tag = tl->fc_tag; in ext4_fc_replay_inode()
1559 inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); in ext4_fc_replay_inode()
1564 inode_len - off_gen); in ext4_fc_replay_inode()
1565 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { in ext4_fc_replay_inode()
1566 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); in ext4_fc_replay_inode()
1567 if (eh->eh_magic != EXT4_EXT_MAGIC) { in ext4_fc_replay_inode()
1569 eh->eh_magic = EXT4_EXT_MAGIC; in ext4_fc_replay_inode()
1570 eh->eh_max = cpu_to_le16( in ext4_fc_replay_inode()
1571 (sizeof(raw_inode->i_block) - in ext4_fc_replay_inode()
1575 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) { in ext4_fc_replay_inode()
1576 memcpy(raw_inode->i_block, raw_fc_inode->i_block, in ext4_fc_replay_inode()
1577 sizeof(raw_inode->i_block)); in ext4_fc_replay_inode()
1595 return -EFSCORRUPTED; in ext4_fc_replay_inode()
1606 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation); in ext4_fc_replay_inode()
1616 blkdev_issue_flush(sb->s_bdev); in ext4_fc_replay_inode()
1650 ret = -EINVAL; in ext4_fc_replay_create()
1654 if (S_ISDIR(inode->i_mode)) { in ext4_fc_replay_create()
1692 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_record_regions()
1697 if (replay && state->fc_regions_used != state->fc_regions_valid) in ext4_fc_record_regions()
1698 state->fc_regions_used = state->fc_regions_valid; in ext4_fc_record_regions()
1699 if (state->fc_regions_used == state->fc_regions_size) { in ext4_fc_record_regions()
1702 fc_regions = krealloc(state->fc_regions, in ext4_fc_record_regions()
1704 (state->fc_regions_size + in ext4_fc_record_regions()
1708 return -ENOMEM; in ext4_fc_record_regions()
1709 state->fc_regions_size += in ext4_fc_record_regions()
1711 state->fc_regions = fc_regions; in ext4_fc_record_regions()
1713 region = &state->fc_regions[state->fc_regions_used++]; in ext4_fc_record_regions()
1714 region->ino = ino; in ext4_fc_record_regions()
1715 region->lblk = lblk; in ext4_fc_record_regions()
1716 region->pblk = pblk; in ext4_fc_record_regions()
1717 region->len = len; in ext4_fc_record_regions()
1720 state->fc_regions_valid++; in ext4_fc_record_regions()
1743 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block), in ext4_fc_replay_add_range()
1752 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); in ext4_fc_replay_add_range()
1756 start = le32_to_cpu(ex->ee_block); in ext4_fc_replay_add_range()
1764 inode->i_ino); in ext4_fc_replay_add_range()
1783 &newex, start_pblk + cur - start); in ext4_fc_replay_add_range()
1787 down_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_add_range()
1790 up_write((&EXT4_I(inode)->i_data_sem)); in ext4_fc_replay_add_range()
1796 if (start_pblk + cur - start != map.m_pblk) { in ext4_fc_replay_add_range()
1804 start_pblk + cur - start); in ext4_fc_replay_add_range()
1816 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); in ext4_fc_replay_add_range()
1835 remaining -= map.m_len; in ext4_fc_replay_add_range()
1838 sb->s_blocksize_bits); in ext4_fc_replay_add_range()
1869 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); in ext4_fc_replay_del_range()
1874 inode->i_ino, le32_to_cpu(lrange.fc_lblk), in ext4_fc_replay_del_range()
1884 remaining -= ret; in ext4_fc_replay_del_range()
1886 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); in ext4_fc_replay_del_range()
1888 remaining -= map.m_len; in ext4_fc_replay_del_range()
1893 down_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_del_range()
1896 le32_to_cpu(lrange.fc_len) - 1); in ext4_fc_replay_del_range()
1897 up_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_del_range()
1901 i_size_read(inode) >> sb->s_blocksize_bits); in ext4_fc_replay_del_range()
1917 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_set_bitmaps_and_counters()
1918 for (i = 0; i < state->fc_modified_inodes_used; i++) { in ext4_fc_set_bitmaps_and_counters()
1919 inode = ext4_iget(sb, state->fc_modified_inodes[i], in ext4_fc_set_bitmaps_and_counters()
1923 state->fc_modified_inodes[i]); in ext4_fc_set_bitmaps_and_counters()
1934 map.m_len = end - cur; in ext4_fc_set_bitmaps_and_counters()
1943 for (j = 0; j < path->p_depth; j++) in ext4_fc_set_bitmaps_and_counters()
1944 ext4_mb_mark_bb(inode->i_sb, in ext4_fc_set_bitmaps_and_counters()
1950 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, in ext4_fc_set_bitmaps_and_counters()
1972 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_replay_check_excluded()
1973 for (i = 0; i < state->fc_regions_valid; i++) { in ext4_fc_replay_check_excluded()
1974 if (state->fc_regions[i].ino == 0 || in ext4_fc_replay_check_excluded()
1975 state->fc_regions[i].len == 0) in ext4_fc_replay_check_excluded()
1977 if (in_range(blk, state->fc_regions[i].pblk, in ext4_fc_replay_check_excluded()
1978 state->fc_regions[i].len)) in ext4_fc_replay_check_excluded()
1989 sbi->s_mount_state &= ~EXT4_FC_REPLAY; in ext4_fc_replay_cleanup()
1990 kfree(sbi->s_fc_replay_state.fc_regions); in ext4_fc_replay_cleanup()
1991 kfree(sbi->s_fc_replay_state.fc_modified_inodes); in ext4_fc_replay_cleanup()
2005 len -= sizeof(struct ext4_fc_dentry_info); in ext4_fc_value_len_isvalid()
2008 len -= sizeof(struct ext4_fc_inode); in ext4_fc_value_len_isvalid()
2010 len <= sbi->s_inode_size; in ext4_fc_value_len_isvalid()
2026 * - Make sure the fast commit area has valid tags for replay
2027 * - Count number of tags that need to be replayed by the replay handler
2028 * - Verify CRC
2029 * - Create a list of excluded blocks for allocation during replay phase
2035 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
2042 struct super_block *sb = journal->j_private; in ext4_fc_replay_scan()
2053 state = &sbi->s_fc_replay_state; in ext4_fc_replay_scan()
2055 start = (u8 *)bh->b_data; in ext4_fc_replay_scan()
2056 end = start + journal->j_blocksize; in ext4_fc_replay_scan()
2058 if (state->fc_replay_expected_off == 0) { in ext4_fc_replay_scan()
2059 state->fc_cur_tag = 0; in ext4_fc_replay_scan()
2060 state->fc_replay_num_tags = 0; in ext4_fc_replay_scan()
2061 state->fc_crc = 0; in ext4_fc_replay_scan()
2062 state->fc_regions = NULL; in ext4_fc_replay_scan()
2063 state->fc_regions_valid = state->fc_regions_used = in ext4_fc_replay_scan()
2064 state->fc_regions_size = 0; in ext4_fc_replay_scan()
2066 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag) in ext4_fc_replay_scan()
2071 if (off != state->fc_replay_expected_off) { in ext4_fc_replay_scan()
2072 ret = -EFSCORRUPTED; in ext4_fc_replay_scan()
2076 state->fc_replay_expected_off++; in ext4_fc_replay_scan()
2077 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; in ext4_fc_replay_scan()
2081 if (tl.fc_len > end - val || in ext4_fc_replay_scan()
2083 ret = state->fc_replay_num_tags ? in ext4_fc_replay_scan()
2084 JBD2_FC_REPLAY_STOP : -ECANCELED; in ext4_fc_replay_scan()
2088 tag2str(tl.fc_tag), bh->b_blocknr); in ext4_fc_replay_scan()
2095 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex), in ext4_fc_replay_scan()
2107 state->fc_cur_tag++; in ext4_fc_replay_scan()
2108 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
2112 state->fc_cur_tag++; in ext4_fc_replay_scan()
2114 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
2119 le32_to_cpu(tail.fc_crc) == state->fc_crc) { in ext4_fc_replay_scan()
2120 state->fc_replay_num_tags = state->fc_cur_tag; in ext4_fc_replay_scan()
2121 state->fc_regions_valid = in ext4_fc_replay_scan()
2122 state->fc_regions_used; in ext4_fc_replay_scan()
2124 ret = state->fc_replay_num_tags ? in ext4_fc_replay_scan()
2125 JBD2_FC_REPLAY_STOP : -EFSBADCRC; in ext4_fc_replay_scan()
2127 state->fc_crc = 0; in ext4_fc_replay_scan()
2133 ret = -EOPNOTSUPP; in ext4_fc_replay_scan()
2140 state->fc_cur_tag++; in ext4_fc_replay_scan()
2141 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
2145 ret = state->fc_replay_num_tags ? in ext4_fc_replay_scan()
2146 JBD2_FC_REPLAY_STOP : -ECANCELED; in ext4_fc_replay_scan()
2164 struct super_block *sb = journal->j_private; in ext4_fc_replay()
2169 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state; in ext4_fc_replay()
2173 state->fc_current_pass = PASS_SCAN; in ext4_fc_replay()
2177 if (state->fc_current_pass != pass) { in ext4_fc_replay()
2178 state->fc_current_pass = pass; in ext4_fc_replay()
2179 sbi->s_mount_state |= EXT4_FC_REPLAY; in ext4_fc_replay()
2181 if (!sbi->s_fc_replay_state.fc_replay_num_tags) { in ext4_fc_replay()
2188 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) { in ext4_fc_replay()
2194 start = (u8 *)bh->b_data; in ext4_fc_replay()
2195 end = start + journal->j_blocksize; in ext4_fc_replay()
2197 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; in ext4_fc_replay()
2202 if (state->fc_replay_num_tags == 0) { in ext4_fc_replay()
2209 state->fc_replay_num_tags--; in ext4_fc_replay()
2243 ret = -ECANCELED; in ext4_fc_replay()
2260 journal->j_fc_replay_callback = ext4_fc_replay; in ext4_fc_init()
2263 journal->j_fc_cleanup_callback = ext4_fc_cleanup; in ext4_fc_init()
2281 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); in ext4_fc_info_show()
2282 struct ext4_fc_stats *stats = &sbi->s_fc_stats; in ext4_fc_info_show()
2289 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", in ext4_fc_info_show()
2290 stats->fc_num_commits, stats->fc_ineligible_commits, in ext4_fc_info_show()
2291 stats->fc_numblks, in ext4_fc_info_show()
2292 div_u64(stats->s_fc_avg_commit_time, 1000)); in ext4_fc_info_show()
2296 stats->fc_ineligible_reason_count[i]); in ext4_fc_info_show()
2307 return -ENOMEM; in ext4_fc_init_dentry_cache()