Lines Matching +full:wait +full:- +full:on +full:- +full:write
1 // SPDX-License-Identifier: GPL-2.0-only
3 * fs/fs-writeback.c
14 * Additions for address_space-based writeback
28 #include <linux/backing-dev.h>
35 * 4MB minimal write chunk size
37 #define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_SHIFT - 10))
51 unsigned int auto_free:1; /* free on completion */
90 set_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_populated()
91 WARN_ON_ONCE(!wb->avg_write_bandwidth); in wb_io_lists_populated()
92 atomic_long_add(wb->avg_write_bandwidth, in wb_io_lists_populated()
93 &wb->bdi->tot_write_bandwidth); in wb_io_lists_populated()
100 if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) && in wb_io_lists_depopulated()
101 list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) { in wb_io_lists_depopulated()
102 clear_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_depopulated()
103 WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth, in wb_io_lists_depopulated()
104 &wb->bdi->tot_write_bandwidth) < 0); in wb_io_lists_depopulated()
109 * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list
112 * @head: one of @wb->b_{dirty|io|more_io|dirty_time}
114 * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.
122 assert_spin_locked(&wb->list_lock); in inode_io_list_move_locked()
123 assert_spin_locked(&inode->i_lock); in inode_io_list_move_locked()
124 WARN_ON_ONCE(inode->i_state & I_FREEING); in inode_io_list_move_locked()
126 list_move(&inode->i_io_list, head); in inode_io_list_move_locked()
129 if (head != &wb->b_dirty_time) in inode_io_list_move_locked()
138 spin_lock_irq(&wb->work_lock); in wb_wakeup()
139 if (test_bit(WB_registered, &wb->state)) in wb_wakeup()
140 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_wakeup()
141 spin_unlock_irq(&wb->work_lock); in wb_wakeup()
147 struct wb_completion *done = work->done; in finish_writeback_work()
149 if (work->auto_free) in finish_writeback_work()
152 wait_queue_head_t *waitq = done->waitq; in finish_writeback_work()
155 if (atomic_dec_and_test(&done->cnt)) in finish_writeback_work()
165 if (work->done) in wb_queue_work()
166 atomic_inc(&work->done->cnt); in wb_queue_work()
168 spin_lock_irq(&wb->work_lock); in wb_queue_work()
170 if (test_bit(WB_registered, &wb->state)) { in wb_queue_work()
171 list_add_tail(&work->list, &wb->work_list); in wb_queue_work()
172 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_queue_work()
176 spin_unlock_irq(&wb->work_lock); in wb_queue_work()
180 * wb_wait_for_completion - wait for completion of bdi_writeback_works
183 * Wait for one or more work items issued to @bdi with their ->done field
187 * automatically on completion.
191 atomic_dec(&done->cnt); /* put down the initial count */ in wb_wait_for_completion()
192 wait_event(*done->waitq, !atomic_read(&done->cnt)); in wb_wait_for_completion()
205 * The current cgroup writeback is built on the assumption that multiple
209 * avoiding too aggressive flip-flops from occasional foreign writes.
221 #define WB_FRN_HIST_SLOTS 16 /* inode->i_wb_frn_history is 16bit */
234 #define WB_MAX_INODES_PER_ISW ((1024UL - sizeof(struct inode_switch_wbs_context)) \
260 wb = &bdi->wb; in __inode_attach_wb()
266 if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) in __inode_attach_wb()
272 * inode_cgwb_move_to_attached - put the inode onto wb->b_attached list
277 * list. Only inodes attached to cgwb's are kept on this list.
282 assert_spin_locked(&wb->list_lock); in inode_cgwb_move_to_attached()
283 assert_spin_locked(&inode->i_lock); in inode_cgwb_move_to_attached()
284 WARN_ON_ONCE(inode->i_state & I_FREEING); in inode_cgwb_move_to_attached()
286 inode->i_state &= ~I_SYNC_QUEUED; in inode_cgwb_move_to_attached()
287 if (wb != &wb->bdi->wb) in inode_cgwb_move_to_attached()
288 list_move(&inode->i_io_list, &wb->b_attached); in inode_cgwb_move_to_attached()
290 list_del_init(&inode->i_io_list); in inode_cgwb_move_to_attached()
295 * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it
298 * Returns @inode's wb with its list_lock held. @inode->i_lock must be
299 * held on entry and is released on return. The returned wb is guaranteed
304 __releases(&inode->i_lock) in locked_inode_to_wb_and_lock_list()
305 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
312 * @inode->i_lock and @wb->list_lock but list_lock nests in locked_inode_to_wb_and_lock_list()
317 spin_unlock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
318 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
321 if (likely(wb == inode->i_wb)) { in locked_inode_to_wb_and_lock_list()
326 spin_unlock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
329 spin_lock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
334 * inode_to_wb_and_lock_list - determine an inode's wb and lock it
337 * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held
338 * on entry.
341 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
343 spin_lock(&inode->i_lock); in inode_to_wb_and_lock_list()
354 * the first part, all inode pointers are placed into a NULL-terminated
356 * an inode could be left in a non-consistent state.
364 down_write(&bdi->wb_switch_rwsem); in bdi_down_write_wb_switch_rwsem()
369 up_write(&bdi->wb_switch_rwsem); in bdi_up_write_wb_switch_rwsem()
376 struct address_space *mapping = inode->i_mapping; in inode_do_switch_wbs()
377 XA_STATE(xas, &mapping->i_pages, 0); in inode_do_switch_wbs()
381 spin_lock(&inode->i_lock); in inode_do_switch_wbs()
382 xa_lock_irq(&mapping->i_pages); in inode_do_switch_wbs()
386 * path owns the inode and we shouldn't modify ->i_io_list. in inode_do_switch_wbs()
388 if (unlikely(inode->i_state & (I_FREEING | I_WILL_FREE))) in inode_do_switch_wbs()
401 wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr); in inode_do_switch_wbs()
410 wb_stat_mod(old_wb, WB_WRITEBACK, -nr); in inode_do_switch_wbs()
415 atomic_dec(&old_wb->writeback_inodes); in inode_do_switch_wbs()
416 atomic_inc(&new_wb->writeback_inodes); in inode_do_switch_wbs()
423 * the specific list @inode was on is ignored and the @inode is put on in inode_do_switch_wbs()
424 * ->b_dirty which is always correct including from ->b_dirty_time. in inode_do_switch_wbs()
425 * The transfer preserves @inode->dirtied_when ordering. If the @inode in inode_do_switch_wbs()
426 * was clean, it means it was on the b_attached list, so move it onto in inode_do_switch_wbs()
429 if (!list_empty(&inode->i_io_list)) { in inode_do_switch_wbs()
430 inode->i_wb = new_wb; in inode_do_switch_wbs()
432 if (inode->i_state & I_DIRTY_ALL) { in inode_do_switch_wbs()
435 list_for_each_entry(pos, &new_wb->b_dirty, i_io_list) in inode_do_switch_wbs()
436 if (time_after_eq(inode->dirtied_when, in inode_do_switch_wbs()
437 pos->dirtied_when)) in inode_do_switch_wbs()
440 pos->i_io_list.prev); in inode_do_switch_wbs()
445 inode->i_wb = new_wb; in inode_do_switch_wbs()
448 /* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */ in inode_do_switch_wbs()
449 inode->i_wb_frn_winner = 0; in inode_do_switch_wbs()
450 inode->i_wb_frn_avg_time = 0; in inode_do_switch_wbs()
451 inode->i_wb_frn_history = 0; in inode_do_switch_wbs()
458 smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); in inode_do_switch_wbs()
460 xa_unlock_irq(&mapping->i_pages); in inode_do_switch_wbs()
461 spin_unlock(&inode->i_lock); in inode_do_switch_wbs()
470 struct backing_dev_info *bdi = inode_to_bdi(isw->inodes[0]); in inode_switch_wbs_work_fn()
471 struct bdi_writeback *old_wb = isw->inodes[0]->i_wb; in inode_switch_wbs_work_fn()
472 struct bdi_writeback *new_wb = isw->new_wb; in inode_switch_wbs_work_fn()
480 down_read(&bdi->wb_switch_rwsem); in inode_switch_wbs_work_fn()
488 * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock in inode_switch_wbs_work_fn()
489 * gives us exclusion against all wb related operations on @inode in inode_switch_wbs_work_fn()
493 spin_lock(&old_wb->list_lock); in inode_switch_wbs_work_fn()
494 spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING); in inode_switch_wbs_work_fn()
496 spin_lock(&new_wb->list_lock); in inode_switch_wbs_work_fn()
497 spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); in inode_switch_wbs_work_fn()
500 for (inodep = isw->inodes; *inodep; inodep++) { in inode_switch_wbs_work_fn()
501 WARN_ON_ONCE((*inodep)->i_wb != old_wb); in inode_switch_wbs_work_fn()
506 spin_unlock(&new_wb->list_lock); in inode_switch_wbs_work_fn()
507 spin_unlock(&old_wb->list_lock); in inode_switch_wbs_work_fn()
509 up_read(&bdi->wb_switch_rwsem); in inode_switch_wbs_work_fn()
516 for (inodep = isw->inodes; *inodep; inodep++) in inode_switch_wbs_work_fn()
538 spin_lock(&inode->i_lock); in inode_prepare_wbs_switch()
539 if (!(inode->i_sb->s_flags & SB_ACTIVE) || in inode_prepare_wbs_switch()
540 inode->i_state & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) || in inode_prepare_wbs_switch()
542 spin_unlock(&inode->i_lock); in inode_prepare_wbs_switch()
545 inode->i_state |= I_WB_SWITCH; in inode_prepare_wbs_switch()
547 spin_unlock(&inode->i_lock); in inode_prepare_wbs_switch()
553 * inode_switch_wbs - change the wb association of an inode
567 if (inode->i_state & I_WB_SWITCH) in inode_switch_wbs()
589 isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); in inode_switch_wbs()
591 if (!isw->new_wb) in inode_switch_wbs()
594 if (!inode_prepare_wbs_switch(inode, isw->new_wb)) in inode_switch_wbs()
597 isw->inodes[0] = inode; in inode_switch_wbs()
605 INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn); in inode_switch_wbs()
606 queue_rcu_work(isw_wq, &isw->work); in inode_switch_wbs()
611 if (isw->new_wb) in inode_switch_wbs()
612 wb_put(isw->new_wb); in inode_switch_wbs()
622 if (!inode_prepare_wbs_switch(inode, isw->new_wb)) in isw_prepare_wbs_switch()
625 isw->inodes[*nr] = inode; in isw_prepare_wbs_switch()
628 if (*nr >= WB_MAX_INODES_PER_ISW - 1) in isw_prepare_wbs_switch()
635 * cleanup_offline_cgwb - detach associated inodes
656 for (memcg_css = wb->memcg_css->parent; memcg_css; in cleanup_offline_cgwb()
657 memcg_css = memcg_css->parent) { in cleanup_offline_cgwb()
658 isw->new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL); in cleanup_offline_cgwb()
659 if (isw->new_wb) in cleanup_offline_cgwb()
662 if (unlikely(!isw->new_wb)) in cleanup_offline_cgwb()
663 isw->new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ in cleanup_offline_cgwb()
666 spin_lock(&wb->list_lock); in cleanup_offline_cgwb()
675 restart = isw_prepare_wbs_switch(isw, &wb->b_attached, &nr); in cleanup_offline_cgwb()
677 restart = isw_prepare_wbs_switch(isw, &wb->b_dirty_time, &nr); in cleanup_offline_cgwb()
678 spin_unlock(&wb->list_lock); in cleanup_offline_cgwb()
683 wb_put(isw->new_wb); in cleanup_offline_cgwb()
694 INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn); in cleanup_offline_cgwb()
695 queue_rcu_work(isw_wq, &isw->work); in cleanup_offline_cgwb()
701 * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it
706 * Record @inode's writeback context into @wbc and unlock the i_lock. On
714 spin_unlock(&inode->i_lock); in wbc_attach_and_unlock_inode()
718 wbc->wb = inode_to_wb(inode); in wbc_attach_and_unlock_inode()
719 wbc->inode = inode; in wbc_attach_and_unlock_inode()
721 wbc->wb_id = wbc->wb->memcg_css->id; in wbc_attach_and_unlock_inode()
722 wbc->wb_lcand_id = inode->i_wb_frn_winner; in wbc_attach_and_unlock_inode()
723 wbc->wb_tcand_id = 0; in wbc_attach_and_unlock_inode()
724 wbc->wb_bytes = 0; in wbc_attach_and_unlock_inode()
725 wbc->wb_lcand_bytes = 0; in wbc_attach_and_unlock_inode()
726 wbc->wb_tcand_bytes = 0; in wbc_attach_and_unlock_inode()
728 wb_get(wbc->wb); in wbc_attach_and_unlock_inode()
729 spin_unlock(&inode->i_lock); in wbc_attach_and_unlock_inode()
738 if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css))) in wbc_attach_and_unlock_inode()
739 inode_switch_wbs(inode, wbc->wb_id); in wbc_attach_and_unlock_inode()
744 * wbc_detach_inode - disassociate wbc from inode and perform foreign detection
750 * As concurrent write sharing of an inode is expected to be very rare and
751 * memcg only tracks page ownership on first-use basis severely confining
753 * per-inode. While the support for concurrent write sharing of an inode
756 * charging only by first-use can too readily lead to grossly incorrect
766 * On each writeback attempt, @wbc tries to detect the majority writer
767 * using Boyer-Moore majority vote algorithm. In addition to the byte
772 * to semi-reliably detect the most active writer even when it's not the
777 * inode->i_wb_frn_history. If the amount of recorded foreign IO time is
782 struct bdi_writeback *wb = wbc->wb; in wbc_detach_inode()
783 struct inode *inode = wbc->inode; in wbc_detach_inode()
791 history = inode->i_wb_frn_history; in wbc_detach_inode()
792 avg_time = inode->i_wb_frn_avg_time; in wbc_detach_inode()
795 if (wbc->wb_bytes >= wbc->wb_lcand_bytes && in wbc_detach_inode()
796 wbc->wb_bytes >= wbc->wb_tcand_bytes) { in wbc_detach_inode()
797 max_id = wbc->wb_id; in wbc_detach_inode()
798 max_bytes = wbc->wb_bytes; in wbc_detach_inode()
799 } else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) { in wbc_detach_inode()
800 max_id = wbc->wb_lcand_id; in wbc_detach_inode()
801 max_bytes = wbc->wb_lcand_bytes; in wbc_detach_inode()
803 max_id = wbc->wb_tcand_id; in wbc_detach_inode()
804 max_bytes = wbc->wb_tcand_bytes; in wbc_detach_inode()
811 * deciding whether to switch or not. This is to prevent one-off in wbc_detach_inode()
815 wb->avg_write_bandwidth); in wbc_detach_inode()
817 avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) - in wbc_detach_inode()
820 avg_time = max_time; /* immediate catch up on first run */ in wbc_detach_inode()
836 if (wbc->wb_id != max_id) in wbc_detach_inode()
837 history |= (1U << slots) - 1; in wbc_detach_inode()
846 * is okay. The main goal is avoiding keeping an inode on in wbc_detach_inode()
857 inode->i_wb_frn_winner = max_id; in wbc_detach_inode()
858 inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX); in wbc_detach_inode()
859 inode->i_wb_frn_history = history; in wbc_detach_inode()
861 wb_put(wbc->wb); in wbc_detach_inode()
862 wbc->wb = NULL; in wbc_detach_inode()
867 * wbc_account_cgroup_owner - account writeback to update inode cgroup ownership
889 if (!wbc->wb || wbc->no_cgroup_owner) in wbc_account_cgroup_owner()
895 if (!(css->flags & CSS_ONLINE)) in wbc_account_cgroup_owner()
898 id = css->id; in wbc_account_cgroup_owner()
900 if (id == wbc->wb_id) { in wbc_account_cgroup_owner()
901 wbc->wb_bytes += bytes; in wbc_account_cgroup_owner()
905 if (id == wbc->wb_lcand_id) in wbc_account_cgroup_owner()
906 wbc->wb_lcand_bytes += bytes; in wbc_account_cgroup_owner()
908 /* Boyer-Moore majority vote algorithm */ in wbc_account_cgroup_owner()
909 if (!wbc->wb_tcand_bytes) in wbc_account_cgroup_owner()
910 wbc->wb_tcand_id = id; in wbc_account_cgroup_owner()
911 if (id == wbc->wb_tcand_id) in wbc_account_cgroup_owner()
912 wbc->wb_tcand_bytes += bytes; in wbc_account_cgroup_owner()
914 wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes); in wbc_account_cgroup_owner()
919 * wb_split_bdi_pages - split nr_pages to write according to bandwidth
921 * @nr_pages: number of pages to write for the whole bdi
923 * Split @wb's portion of @nr_pages according to @wb's write bandwidth in
924 * relation to the total write bandwidth of all wb's w/ dirty inodes on
925 * @wb->bdi.
929 unsigned long this_bw = wb->avg_write_bandwidth; in wb_split_bdi_pages()
930 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); in wb_split_bdi_pages()
936 * This may be called on clean wb's and proportional distribution in wb_split_bdi_pages()
938 * cases. In general, we wanna err on the side of writing more. in wb_split_bdi_pages()
947 * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
953 * have dirty inodes. If @base_work->nr_page isn't %LONG_MAX, it's
955 * total active write bandwidth of @bdi.
962 struct bdi_writeback *wb = list_entry(&bdi->wb_list, in bdi_split_work_to_wbs()
968 list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) { in bdi_split_work_to_wbs()
981 (base_work->sync_mode == WB_SYNC_NONE || in bdi_split_work_to_wbs()
982 list_empty(&wb->b_dirty_time))) in bdi_split_work_to_wbs()
987 nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages); in bdi_split_work_to_wbs()
992 work->nr_pages = nr_pages; in bdi_split_work_to_wbs()
993 work->auto_free = 1; in bdi_split_work_to_wbs()
1001 * Pin @wb so that it stays on @bdi->wb_list. This allows in bdi_split_work_to_wbs()
1008 /* alloc failed, execute synchronously using on-stack fallback */ in bdi_split_work_to_wbs()
1011 work->nr_pages = nr_pages; in bdi_split_work_to_wbs()
1012 work->auto_free = 0; in bdi_split_work_to_wbs()
1013 work->done = &fallback_work_done; in bdi_split_work_to_wbs()
1029 * cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs
1051 return -ENOENT; in cgroup_writeback_by_id()
1059 ret = -ENOENT; in cgroup_writeback_by_id()
1069 ret = -ENOENT; in cgroup_writeback_by_id()
1074 * The caller is attempting to write out most of in cgroup_writeback_by_id()
1080 * BTW the memcg stats are flushed periodically and this is best-effort in cgroup_writeback_by_id()
1089 work->nr_pages = dirty; in cgroup_writeback_by_id()
1090 work->sync_mode = WB_SYNC_NONE; in cgroup_writeback_by_id()
1091 work->range_cyclic = 1; in cgroup_writeback_by_id()
1092 work->reason = reason; in cgroup_writeback_by_id()
1093 work->done = done; in cgroup_writeback_by_id()
1094 work->auto_free = 1; in cgroup_writeback_by_id()
1098 ret = -ENOMEM; in cgroup_writeback_by_id()
1110 * cgroup_writeback_umount - flush inode wb switches for umount
1113 * flushes in-flight inode wb switches. An inode wb switch goes through
1129 * Use rcu_barrier() to wait for all pending callbacks to in cgroup_writeback_umount()
1130 * ensure that all in-flight wb switches are in the workqueue. in cgroup_writeback_umount()
1141 return -ENOMEM; in cgroup_writeback_init()
1154 assert_spin_locked(&wb->list_lock); in inode_cgwb_move_to_attached()
1155 assert_spin_locked(&inode->i_lock); in inode_cgwb_move_to_attached()
1156 WARN_ON_ONCE(inode->i_state & I_FREEING); in inode_cgwb_move_to_attached()
1158 inode->i_state &= ~I_SYNC_QUEUED; in inode_cgwb_move_to_attached()
1159 list_del_init(&inode->i_io_list); in inode_cgwb_move_to_attached()
1165 __releases(&inode->i_lock) in locked_inode_to_wb_and_lock_list()
1166 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
1170 spin_unlock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
1171 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
1176 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
1180 spin_lock(&wb->list_lock); in inode_to_wb_and_lock_list()
1195 if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) { in bdi_split_work_to_wbs()
1196 base_work->auto_free = 0; in bdi_split_work_to_wbs()
1197 wb_queue_work(&bdi->wb, base_work); in bdi_split_work_to_wbs()
1205 * write can dirty pagecache in the underlying blockdev.
1226 if (test_bit(WB_start_all, &wb->state) || in wb_start_writeback()
1227 test_and_set_bit(WB_start_all, &wb->state)) in wb_start_writeback()
1230 wb->start_all_reason = reason; in wb_start_writeback()
1235 * wb_start_background_writeback - start background writeback
1236 * @wb: bdi_writback to write from
1255 * Remove the inode from the writeback list it is on.
1262 spin_lock(&inode->i_lock); in inode_io_list_del()
1264 inode->i_state &= ~I_SYNC_QUEUED; in inode_io_list_del()
1265 list_del_init(&inode->i_io_list); in inode_io_list_del()
1268 spin_unlock(&inode->i_lock); in inode_io_list_del()
1269 spin_unlock(&wb->list_lock); in inode_io_list_del()
1274 * mark an inode as under writeback on the sb
1278 struct super_block *sb = inode->i_sb; in sb_mark_inode_writeback()
1281 if (list_empty(&inode->i_wb_list)) { in sb_mark_inode_writeback()
1282 spin_lock_irqsave(&sb->s_inode_wblist_lock, flags); in sb_mark_inode_writeback()
1283 if (list_empty(&inode->i_wb_list)) { in sb_mark_inode_writeback()
1284 list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb); in sb_mark_inode_writeback()
1287 spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags); in sb_mark_inode_writeback()
1292 * clear an inode as under writeback on the sb
1296 struct super_block *sb = inode->i_sb; in sb_clear_inode_writeback()
1299 if (!list_empty(&inode->i_wb_list)) { in sb_clear_inode_writeback()
1300 spin_lock_irqsave(&sb->s_inode_wblist_lock, flags); in sb_clear_inode_writeback()
1301 if (!list_empty(&inode->i_wb_list)) { in sb_clear_inode_writeback()
1302 list_del_init(&inode->i_wb_list); in sb_clear_inode_writeback()
1305 spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags); in sb_clear_inode_writeback()
1310 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
1311 * furthest end of its superblock's dirty-inode list.
1313 * Before stamping the inode's ->dirtied_when, we check to see whether it is
1314 * already the most-recently-dirtied inode on the b_dirty list. If that is
1320 assert_spin_locked(&inode->i_lock); in redirty_tail_locked()
1322 inode->i_state &= ~I_SYNC_QUEUED; in redirty_tail_locked()
1328 if (inode->i_state & I_FREEING) { in redirty_tail_locked()
1329 list_del_init(&inode->i_io_list); in redirty_tail_locked()
1333 if (!list_empty(&wb->b_dirty)) { in redirty_tail_locked()
1336 tail = wb_inode(wb->b_dirty.next); in redirty_tail_locked()
1337 if (time_before(inode->dirtied_when, tail->dirtied_when)) in redirty_tail_locked()
1338 inode->dirtied_when = jiffies; in redirty_tail_locked()
1340 inode_io_list_move_locked(inode, wb, &wb->b_dirty); in redirty_tail_locked()
1345 spin_lock(&inode->i_lock); in redirty_tail()
1347 spin_unlock(&inode->i_lock); in redirty_tail()
1351 * requeue inode for re-scanning after bdi->b_io list is exhausted.
1355 inode_io_list_move_locked(inode, wb, &wb->b_more_io); in requeue_io()
1360 inode->i_state &= ~I_SYNC; in inode_sync_complete()
1365 wake_up_bit(&inode->i_state, __I_SYNC); in inode_sync_complete()
1370 bool ret = time_after(inode->dirtied_when, t); in inode_dirtied_after()
1375 * This test is necessary to prevent such wrapped-around relative times in inode_dirtied_after()
1378 ret = ret && time_before_eq(inode->dirtied_when, jiffies); in inode_dirtied_after()
1399 inode = wb_inode(delaying_queue->prev); in move_expired_inodes()
1402 spin_lock(&inode->i_lock); in move_expired_inodes()
1403 list_move(&inode->i_io_list, &tmp); in move_expired_inodes()
1405 inode->i_state |= I_SYNC_QUEUED; in move_expired_inodes()
1406 spin_unlock(&inode->i_lock); in move_expired_inodes()
1407 if (sb_is_blkdev_sb(inode->i_sb)) in move_expired_inodes()
1409 if (sb && sb != inode->i_sb) in move_expired_inodes()
1411 sb = inode->i_sb; in move_expired_inodes()
1422 * we don't take inode->i_lock here because it is just a pointless overhead. in move_expired_inodes()
1427 sb = wb_inode(tmp.prev)->i_sb; in move_expired_inodes()
1430 if (inode->i_sb == sb) in move_expired_inodes()
1431 list_move(&inode->i_io_list, dispatch_queue); in move_expired_inodes()
1447 * +--> dequeue for IO
1455 assert_spin_locked(&wb->list_lock); in queue_io()
1456 list_splice_init(&wb->b_more_io, &wb->b_io); in queue_io()
1457 moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before); in queue_io()
1458 if (!work->for_sync) in queue_io()
1459 time_expire_jif = jiffies - dirtytime_expire_interval * HZ; in queue_io()
1460 moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, in queue_io()
1471 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) { in write_inode()
1473 ret = inode->i_sb->s_op->write_inode(inode, wbc); in write_inode()
1481 * Wait for writeback on an inode to complete. Called with i_lock held.
1485 __releases(inode->i_lock) in __inode_wait_for_writeback()
1486 __acquires(inode->i_lock) in __inode_wait_for_writeback()
1488 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC); in __inode_wait_for_writeback()
1491 wqh = bit_waitqueue(&inode->i_state, __I_SYNC); in __inode_wait_for_writeback()
1492 while (inode->i_state & I_SYNC) { in __inode_wait_for_writeback()
1493 spin_unlock(&inode->i_lock); in __inode_wait_for_writeback()
1496 spin_lock(&inode->i_lock); in __inode_wait_for_writeback()
1501 * Wait for writeback on an inode to complete. Caller must have inode pinned.
1505 spin_lock(&inode->i_lock); in inode_wait_for_writeback()
1507 spin_unlock(&inode->i_lock); in inode_wait_for_writeback()
1516 __releases(inode->i_lock) in inode_sleep_on_writeback()
1518 DEFINE_WAIT(wait); in inode_sleep_on_writeback()
1519 wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC); in inode_sleep_on_writeback()
1522 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); in inode_sleep_on_writeback()
1523 sleep = inode->i_state & I_SYNC; in inode_sleep_on_writeback()
1524 spin_unlock(&inode->i_lock); in inode_sleep_on_writeback()
1527 finish_wait(wqh, &wait); in inode_sleep_on_writeback()
1531 * Find proper writeback list for the inode depending on its current state and
1534 * inodes. This function can be called only by flusher thread - noone else
1541 if (inode->i_state & I_FREEING) in requeue_inode()
1549 if ((inode->i_state & I_DIRTY) && in requeue_inode()
1550 (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)) in requeue_inode()
1551 inode->dirtied_when = jiffies; in requeue_inode()
1553 if (wbc->pages_skipped) { in requeue_inode()
1560 if (inode->i_state & I_DIRTY_ALL) in requeue_inode()
1567 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) { in requeue_inode()
1569 * We didn't write back all the pages. nfs_writepages() in requeue_inode()
1572 if (wbc->nr_to_write <= 0) { in requeue_inode()
1579 * avoid spinning on the CPU (100% iowait) in requeue_inode()
1585 } else if (inode->i_state & I_DIRTY) { in requeue_inode()
1592 } else if (inode->i_state & I_DIRTY_TIME) { in requeue_inode()
1593 inode->dirtied_when = jiffies; in requeue_inode()
1594 inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); in requeue_inode()
1595 inode->i_state &= ~I_SYNC_QUEUED; in requeue_inode()
1603 * Write out an inode and its dirty pages (or some of its dirty pages, depending
1604 * on @wbc->nr_to_write), and clear the relevant dirty flags from i_state.
1606 * This doesn't remove the inode from the writeback list it is on, except
1616 struct address_space *mapping = inode->i_mapping; in __writeback_single_inode()
1617 long nr_to_write = wbc->nr_to_write; in __writeback_single_inode()
1621 WARN_ON(!(inode->i_state & I_SYNC)); in __writeback_single_inode()
1628 * Make sure to wait on the data before writing out the metadata. in __writeback_single_inode()
1629 * This is important for filesystems that modify metadata on data in __writeback_single_inode()
1631 * separate, external IO completion path and ->sync_fs for guaranteeing in __writeback_single_inode()
1634 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) { in __writeback_single_inode()
1641 * If the inode has dirty timestamps and we need to write them, call in __writeback_single_inode()
1645 if ((inode->i_state & I_DIRTY_TIME) && in __writeback_single_inode()
1646 (wbc->sync_mode == WB_SYNC_ALL || in __writeback_single_inode()
1647 time_after(jiffies, inode->dirtied_time_when + in __writeback_single_inode()
1659 spin_lock(&inode->i_lock); in __writeback_single_inode()
1660 dirty = inode->i_state & I_DIRTY; in __writeback_single_inode()
1661 inode->i_state &= ~dirty; in __writeback_single_inode()
1665 * __mark_inode_dirty() to test i_state without grabbing i_lock - in __writeback_single_inode()
1677 inode->i_state |= I_DIRTY_PAGES; in __writeback_single_inode()
1678 else if (unlikely(inode->i_state & I_PINNING_NETFS_WB)) { in __writeback_single_inode()
1679 if (!(inode->i_state & I_DIRTY_PAGES)) { in __writeback_single_inode()
1680 inode->i_state &= ~I_PINNING_NETFS_WB; in __writeback_single_inode()
1681 wbc->unpinned_netfs_wb = true; in __writeback_single_inode()
1686 spin_unlock(&inode->i_lock); in __writeback_single_inode()
1688 /* Don't write the inode if only I_DIRTY_PAGES was set */ in __writeback_single_inode()
1694 wbc->unpinned_netfs_wb = false; in __writeback_single_inode()
1700 * Write out an inode's dirty data and metadata on-demand, i.e. separately from
1702 * writeback_sb_inodes(). @wbc controls various aspects of the write, such as
1703 * whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE).
1714 spin_lock(&inode->i_lock); in writeback_single_inode()
1715 if (!atomic_read(&inode->i_count)) in writeback_single_inode()
1716 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING))); in writeback_single_inode()
1718 WARN_ON(inode->i_state & I_WILL_FREE); in writeback_single_inode()
1720 if (inode->i_state & I_SYNC) { in writeback_single_inode()
1722 * Writeback is already running on the inode. For WB_SYNC_NONE, in writeback_single_inode()
1724 * must wait for the existing writeback to complete, then do in writeback_single_inode()
1727 if (wbc->sync_mode != WB_SYNC_ALL) in writeback_single_inode()
1731 WARN_ON(inode->i_state & I_SYNC); in writeback_single_inode()
1735 * For data-integrity syncs we also need to check whether any pages are in writeback_single_inode()
1737 * there are any such pages, we'll need to wait for them. in writeback_single_inode()
1739 if (!(inode->i_state & I_DIRTY_ALL) && in writeback_single_inode()
1740 (wbc->sync_mode != WB_SYNC_ALL || in writeback_single_inode()
1741 !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))) in writeback_single_inode()
1743 inode->i_state |= I_SYNC; in writeback_single_inode()
1751 spin_lock(&inode->i_lock); in writeback_single_inode()
1756 if (!(inode->i_state & I_FREEING)) { in writeback_single_inode()
1762 if (!(inode->i_state & I_DIRTY_ALL)) in writeback_single_inode()
1764 else if (!(inode->i_state & I_SYNC_QUEUED)) { in writeback_single_inode()
1765 if ((inode->i_state & I_DIRTY)) in writeback_single_inode()
1767 else if (inode->i_state & I_DIRTY_TIME) { in writeback_single_inode()
1768 inode->dirtied_when = jiffies; in writeback_single_inode()
1771 &wb->b_dirty_time); in writeback_single_inode()
1776 spin_unlock(&wb->list_lock); in writeback_single_inode()
1779 spin_unlock(&inode->i_lock); in writeback_single_inode()
1801 if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages) in writeback_chunk_size()
1804 pages = min(wb->avg_write_bandwidth / 2, in writeback_chunk_size()
1806 pages = min(pages, work->nr_pages); in writeback_chunk_size()
1815 * Write a portion of b_io inodes which belong to @sb.
1819 * NOTE! This is called with wb->list_lock held, and will
1828 .sync_mode = work->sync_mode, in writeback_sb_inodes()
1829 .tagged_writepages = work->tagged_writepages, in writeback_sb_inodes()
1830 .for_kupdate = work->for_kupdate, in writeback_sb_inodes()
1831 .for_background = work->for_background, in writeback_sb_inodes()
1832 .for_sync = work->for_sync, in writeback_sb_inodes()
1833 .range_cyclic = work->range_cyclic, in writeback_sb_inodes()
1841 while (!list_empty(&wb->b_io)) { in writeback_sb_inodes()
1842 struct inode *inode = wb_inode(wb->b_io.prev); in writeback_sb_inodes()
1846 if (inode->i_sb != sb) { in writeback_sb_inodes()
1847 if (work->sb) { in writeback_sb_inodes()
1849 * We only want to write back data for this in writeback_sb_inodes()
1870 spin_lock(&inode->i_lock); in writeback_sb_inodes()
1871 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) { in writeback_sb_inodes()
1873 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1876 if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) { in writeback_sb_inodes()
1879 * doing writeback-for-data-integrity, move it to in writeback_sb_inodes()
1881 * other inodes on s_io. in writeback_sb_inodes()
1887 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1891 spin_unlock(&wb->list_lock); in writeback_sb_inodes()
1898 if (inode->i_state & I_SYNC) { in writeback_sb_inodes()
1899 /* Wait for I_SYNC. This function drops i_lock... */ in writeback_sb_inodes()
1902 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1905 inode->i_state |= I_SYNC; in writeback_sb_inodes()
1914 * evict_inode() will wait so the inode cannot be freed. in writeback_sb_inodes()
1919 work->nr_pages -= write_chunk - wbc.nr_to_write; in writeback_sb_inodes()
1920 wrote = write_chunk - wbc.nr_to_write - wbc.pages_skipped; in writeback_sb_inodes()
1933 blk_flush_plug(current->plug, false); in writeback_sb_inodes()
1942 spin_lock(&inode->i_lock); in writeback_sb_inodes()
1943 if (!(inode->i_state & I_DIRTY_ALL)) in writeback_sb_inodes()
1947 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1950 spin_unlock(&tmp_wb->list_lock); in writeback_sb_inodes()
1951 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1961 if (work->nr_pages <= 0) in writeback_sb_inodes()
1974 while (!list_empty(&wb->b_io)) { in __writeback_inodes_wb()
1975 struct inode *inode = wb_inode(wb->b_io.prev); in __writeback_inodes_wb()
1976 struct super_block *sb = inode->i_sb; in __writeback_inodes_wb()
1988 up_read(&sb->s_umount); in __writeback_inodes_wb()
1994 if (work->nr_pages <= 0) in __writeback_inodes_wb()
1998 /* Leave any unwritten inodes on b_io */ in __writeback_inodes_wb()
2014 spin_lock(&wb->list_lock); in writeback_inodes_wb()
2015 if (list_empty(&wb->b_io)) in writeback_inodes_wb()
2018 spin_unlock(&wb->list_lock); in writeback_inodes_wb()
2021 return nr_pages - work.nr_pages; in writeback_inodes_wb()
2028 * dirtying-time in the inode's address_space. So this periodic writeback code
2034 * one-second gap.
2036 * dirtied_before takes precedence over nr_to_write. So we'll only write back
2042 long nr_pages = work->nr_pages; in wb_writeback()
2053 if (work->nr_pages <= 0) in wb_writeback()
2057 * Background writeout and kupdate-style writeback may in wb_writeback()
2062 if ((work->for_background || work->for_kupdate) && in wb_writeback()
2063 !list_empty(&wb->work_list)) in wb_writeback()
2070 if (work->for_background && !wb_over_bg_thresh(wb)) in wb_writeback()
2074 spin_lock(&wb->list_lock); in wb_writeback()
2082 if (work->for_kupdate) { in wb_writeback()
2083 dirtied_before = jiffies - in wb_writeback()
2085 } else if (work->for_background) in wb_writeback()
2089 if (list_empty(&wb->b_io)) in wb_writeback()
2091 if (work->sb) in wb_writeback()
2092 progress = writeback_sb_inodes(work->sb, wb, work); in wb_writeback()
2098 * Did we write something? Try for more in wb_writeback()
2103 * as made some progress on cleaning pages or inodes. in wb_writeback()
2106 spin_unlock(&wb->list_lock); in wb_writeback()
2113 if (list_empty(&wb->b_more_io)) { in wb_writeback()
2114 spin_unlock(&wb->list_lock); in wb_writeback()
2119 * Nothing written. Wait for some inode to in wb_writeback()
2124 inode = wb_inode(wb->b_more_io.prev); in wb_writeback()
2125 spin_lock(&inode->i_lock); in wb_writeback()
2126 spin_unlock(&wb->list_lock); in wb_writeback()
2132 return nr_pages - work->nr_pages; in wb_writeback()
2142 spin_lock_irq(&wb->work_lock); in get_next_work_item()
2143 if (!list_empty(&wb->work_list)) { in get_next_work_item()
2144 work = list_entry(wb->work_list.next, in get_next_work_item()
2146 list_del_init(&work->list); in get_next_work_item()
2148 spin_unlock_irq(&wb->work_lock); in get_next_work_item()
2181 expired = wb->last_old_flush + in wb_check_old_data_flush()
2186 wb->last_old_flush = jiffies; in wb_check_old_data_flush()
2208 if (!test_bit(WB_start_all, &wb->state)) in wb_check_start_all()
2217 .reason = wb->start_all_reason, in wb_check_start_all()
2223 clear_bit(WB_start_all, &wb->state); in wb_check_start_all()
2236 set_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
2244 * Check for a flush-everything request in wb_do_writeback()
2253 clear_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
2268 set_worker_desc("flush-%s", bdi_dev_name(wb->bdi)); in wb_workfn()
2271 !test_bit(WB_registered, &wb->state))) { in wb_workfn()
2281 } while (!list_empty(&wb->work_list)); in wb_workfn()
2293 if (!list_empty(&wb->work_list)) in wb_workfn()
2300 * Start writeback of `nr_pages' pages on this bdi. If `nr_pages' is zero,
2301 * write back the whole world.
2311 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in __wakeup_flusher_threads_bdi()
2333 blk_flush_plug(current->plug, true); in wakeup_flusher_threads()
2346 * inodes on the system. So instead we define a separate delayed work
2350 * If there is any other write activity going on in the file system,
2352 * happened on the file system is a dirtytime inode caused by an atime
2367 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in wakeup_dirtytime_writeback()
2368 if (!list_empty(&wb->b_dirty_time)) in wakeup_dirtytime_writeback()
2382 int dirtytime_interval_handler(struct ctl_table *table, int write, in dirtytime_interval_handler() argument
2387 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); in dirtytime_interval_handler()
2388 if (ret == 0 && write) in dirtytime_interval_handler()
2394 * __mark_inode_dirty - internal function to mark an inode dirty
2413 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
2414 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
2415 * the kernel-internal blockdev inode represents the dirtying time of the
2417 * page->mapping->host, so the page-dirtying time is recorded in the internal
2422 struct super_block *sb = inode->i_sb; in __mark_inode_dirty()
2430 * Inode timestamp update will piggback on this dirtying. in __mark_inode_dirty()
2431 * We tell ->dirty_inode callback that timestamps need to in __mark_inode_dirty()
2434 if (inode->i_state & I_DIRTY_TIME) { in __mark_inode_dirty()
2435 spin_lock(&inode->i_lock); in __mark_inode_dirty()
2436 if (inode->i_state & I_DIRTY_TIME) { in __mark_inode_dirty()
2437 inode->i_state &= ~I_DIRTY_TIME; in __mark_inode_dirty()
2440 spin_unlock(&inode->i_lock); in __mark_inode_dirty()
2445 * (if needed) it can update on-disk fields and journal the in __mark_inode_dirty()
2451 if (sb->s_op->dirty_inode) in __mark_inode_dirty()
2452 sb->s_op->dirty_inode(inode, in __mark_inode_dirty()
2474 if ((inode->i_state & flags) == flags) in __mark_inode_dirty()
2477 spin_lock(&inode->i_lock); in __mark_inode_dirty()
2478 if ((inode->i_state & flags) != flags) { in __mark_inode_dirty()
2479 const int was_dirty = inode->i_state & I_DIRTY; in __mark_inode_dirty()
2483 inode->i_state |= flags; in __mark_inode_dirty()
2493 spin_lock(&inode->i_lock); in __mark_inode_dirty()
2499 * the inode it will place it on the appropriate superblock in __mark_inode_dirty()
2502 if (inode->i_state & I_SYNC_QUEUED) in __mark_inode_dirty()
2509 if (!S_ISBLK(inode->i_mode)) { in __mark_inode_dirty()
2513 if (inode->i_state & I_FREEING) in __mark_inode_dirty()
2517 * If the inode was already on b_dirty/b_io/b_more_io, don't in __mark_inode_dirty()
2518 * reposition it (that would break b_dirty time-ordering). in __mark_inode_dirty()
2524 inode->dirtied_when = jiffies; in __mark_inode_dirty()
2526 inode->dirtied_time_when = jiffies; in __mark_inode_dirty()
2528 if (inode->i_state & I_DIRTY) in __mark_inode_dirty()
2529 dirty_list = &wb->b_dirty; in __mark_inode_dirty()
2531 dirty_list = &wb->b_dirty_time; in __mark_inode_dirty()
2536 spin_unlock(&wb->list_lock); in __mark_inode_dirty()
2537 spin_unlock(&inode->i_lock); in __mark_inode_dirty()
2542 * we have to wake-up the corresponding bdi thread in __mark_inode_dirty()
2543 * to make sure background write-back happens in __mark_inode_dirty()
2547 (wb->bdi->capabilities & BDI_CAP_WRITEBACK)) in __mark_inode_dirty()
2554 spin_unlock(&wb->list_lock); in __mark_inode_dirty()
2555 spin_unlock(&inode->i_lock); in __mark_inode_dirty()
2562 * Concurrent callers will block on the s_sync_lock rather than doing contending
2576 WARN_ON(!rwsem_is_locked(&sb->s_umount)); in wait_sb_inodes()
2578 mutex_lock(&sb->s_sync_lock); in wait_sb_inodes()
2581 * Splice the writeback list onto a temporary list to avoid waiting on in wait_sb_inodes()
2585 * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as in wait_sb_inodes()
2590 spin_lock_irq(&sb->s_inode_wblist_lock); in wait_sb_inodes()
2591 list_splice_init(&sb->s_inodes_wb, &sync_list); in wait_sb_inodes()
2594 * Data integrity sync. Must wait for all pages under writeback, because in wait_sb_inodes()
2596 * writeout started before we write it out. In which case, the inode in wait_sb_inodes()
2597 * may not be on the dirty list, but we still have to wait for that in wait_sb_inodes()
2603 struct address_space *mapping = inode->i_mapping; in wait_sb_inodes()
2611 list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb); in wait_sb_inodes()
2614 * The mapping can appear untagged while still on-list since we in wait_sb_inodes()
2621 spin_unlock_irq(&sb->s_inode_wblist_lock); in wait_sb_inodes()
2623 spin_lock(&inode->i_lock); in wait_sb_inodes()
2624 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) { in wait_sb_inodes()
2625 spin_unlock(&inode->i_lock); in wait_sb_inodes()
2627 spin_lock_irq(&sb->s_inode_wblist_lock); in wait_sb_inodes()
2631 spin_unlock(&inode->i_lock); in wait_sb_inodes()
2646 spin_lock_irq(&sb->s_inode_wblist_lock); in wait_sb_inodes()
2648 spin_unlock_irq(&sb->s_inode_wblist_lock); in wait_sb_inodes()
2650 mutex_unlock(&sb->s_sync_lock); in wait_sb_inodes()
2656 struct backing_dev_info *bdi = sb->s_bdi; in __writeback_inodes_sb_nr()
2669 WARN_ON(!rwsem_is_locked(&sb->s_umount)); in __writeback_inodes_sb_nr()
2671 bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy); in __writeback_inodes_sb_nr()
2676 * writeback_inodes_sb_nr - writeback dirty inodes from given super_block
2678 * @nr: the number of pages to write
2681 * Start writeback on some inodes on this super_block. No guarantees are made
2682 * on how many (if any) will be written, and this function does not wait
2694 * writeback_inodes_sb - writeback dirty inodes from given super_block
2698 * Start writeback on some inodes on this super_block. No guarantees are made
2699 * on how many (if any) will be written, and this function does not wait
2709 * try_to_writeback_inodes_sb - try to start writeback if none underway
2717 if (!down_read_trylock(&sb->s_umount)) in try_to_writeback_inodes_sb()
2721 up_read(&sb->s_umount); in try_to_writeback_inodes_sb()
2726 * sync_inodes_sb - sync sb inode pages
2729 * This function writes and waits on any dirty inode belonging to this
2734 struct backing_dev_info *bdi = sb->s_bdi; in sync_inodes_sb()
2747 * Can't skip on !bdi_has_dirty() because we should wait for !dirty in sync_inodes_sb()
2753 WARN_ON(!rwsem_is_locked(&sb->s_umount)); in sync_inodes_sb()
2766 * write_inode_now - write an inode to disk
2767 * @inode: inode to write to disk
2768 * @sync: whether the write should be synchronous or not
2773 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
2784 if (!mapping_can_writeback(inode->i_mapping)) in write_inode_now()
2793 * sync_inode_metadata - write an inode to disk
2795 * @wait: wait for I/O to complete.
2797 * Write an inode to disk and adjust its dirty state after completion.
2801 int sync_inode_metadata(struct inode *inode, int wait) in sync_inode_metadata() argument
2804 .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE, in sync_inode_metadata()
2805 .nr_to_write = 0, /* metadata-only */ in sync_inode_metadata()